From 28e18749ff02d856de533f165e950a0b2d31aae4 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sat, 10 Aug 2019 22:39:19 -0300 Subject: [PATCH 001/274] WIP: Reimplementation of SIMILAR TO and SUBSTRING...SIMILAR using Google's re2 library. This should fix: CORE-4874 CORE-5664 CORE-3858 CORE-6088 CORE-3773 CORE-5931 CORE-6088 CORE-4893 --- builds/posix/make.defaults | 3 +- doc/sql.extensions/README.similar_to.txt | 36 +- src/burp/burp.cpp | 53 +- src/burp/burp.h | 25 +- src/common/SimilarToRegex.cpp | 821 ++++++++ src/common/SimilarToRegex.h | 75 + src/common/TextType.cpp | 27 - src/common/TextType.h | 39 - src/common/unicode_util.cpp | 31 + src/common/unicode_util.h | 2 + src/dsql/BoolNodes.cpp | 8 +- src/dsql/ExprNodes.cpp | 4 +- src/jrd/Collation.cpp | 183 +- src/jrd/Collation.h | 6 +- src/jrd/IntlManager.cpp | 1 + src/jrd/SimilarToMatcher.h | 1919 ------------------- src/jrd/intl_classes.h | 35 +- src/jrd/replication/Manager.cpp | 55 +- src/jrd/replication/Manager.h | 13 +- src/jrd/validation.cpp | 39 +- src/jrd/validation.h | 9 +- src/utilities/CMakeLists.txt | 7 +- src/utilities/ntrace/TraceConfiguration.cpp | 57 +- src/utilities/ntrace/TracePluginImpl.cpp | 41 +- src/utilities/ntrace/TracePluginImpl.h | 11 +- src/utilities/ntrace/TraceUnicodeUtils.cpp | 65 - src/utilities/ntrace/TraceUnicodeUtils.h | 57 - 27 files changed, 1221 insertions(+), 2401 deletions(-) create mode 100644 src/common/SimilarToRegex.cpp create mode 100644 src/common/SimilarToRegex.h delete mode 100644 src/jrd/SimilarToMatcher.h delete mode 100644 src/utilities/ntrace/TraceUnicodeUtils.cpp delete mode 100644 src/utilities/ntrace/TraceUnicodeUtils.h diff --git a/builds/posix/make.defaults b/builds/posix/make.defaults index f2f62de76f..8351fc4579 100755 --- a/builds/posix/make.defaults +++ b/builds/posix/make.defaults @@ -307,7 +307,8 @@ endif LIB_PATH_OPTS = $(call LIB_LINK_RPATH,lib) $(call LIB_LINK_RPATH,intl) LIB_LINK_SONAME= -Wl,-soname,$(1) LIB_LINK_MAPFILE= -Wl,--version-script,$(1) -FIREBIRD_LIBRARY_LINK= -L$(LIB) -lfbclient $(MATHLIB) $(CRYPTLIB) +# FIXME: +FIREBIRD_LIBRARY_LINK= -L$(LIB) -lfbclient $(MATHLIB) $(CRYPTLIB) -lre2 EXE_LINK_OPTIONS= $(LDFLAGS) $(THR_FLAGS) $(UNDEF_FLAGS) $(LIB_PATH_OPTS) $(LINK_EMPTY_SYMBOLS) LIB_LINK_OPTIONS= $(LDFLAGS) $(THR_FLAGS) -shared diff --git a/doc/sql.extensions/README.similar_to.txt b/doc/sql.extensions/README.similar_to.txt index 2fdac63b29..6dc2c50d71 100644 --- a/doc/sql.extensions/README.similar_to.txt +++ b/doc/sql.extensions/README.similar_to.txt @@ -90,6 +90,7 @@ Note: , , , , , , , , , , or . +3) Since FB 4 the repeat factor low/high values could not be greater than 1000. Syntax description and examples: @@ -174,7 +175,7 @@ Matches a character not identical to one of : Matches a character identical to one of but not identical to one of : - ... ... + ... ... '3' SIMILAR TO '[[:DIGIT:]^3]' -- false '4' SIMILAR TO '[[:DIGIT:]^3]' -- true @@ -220,3 +221,36 @@ insert into department values ('600', 'Engineering', '(408) 555-123'); -- check select * from department where phone not similar to '\([0-9]{3}\) 555\-%' escape '\'; + +Appendice: + +Since FB 4 SIMILAR TO and SUBSTRING...SIMILAR are implemented using the re2 library, +which has the following license: + +Copyright (c) 2009 The RE2 Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/burp/burp.cpp b/src/burp/burp.cpp index b1cc4a392f..75880da7ed 100644 --- a/src/burp/burp.cpp +++ b/src/burp/burp.cpp @@ -2542,15 +2542,11 @@ void BurpGlobals::setupSkipData(const Firebird::string& regexp) ISC_systemToUtf8(filter); BurpGlobals* tdgbl = BurpGlobals::getSpecific(); - if (!unicodeCollation) - unicodeCollation = FB_NEW_POOL(tdgbl->getPool()) UnicodeCollationHolder(tdgbl->getPool()); - Jrd::TextType* const textType = unicodeCollation->getTextType(); - - skipDataMatcher.reset(FB_NEW_POOL(tdgbl->getPool()) - Firebird::SimilarToMatcher > - (tdgbl->getPool(), textType, (const UCHAR*) filter.c_str(), - filter.length(), '\\', true)); + skipDataMatcher.reset(FB_NEW_POOL(tdgbl->getPool()) Firebird::SimilarToRegex( + tdgbl->getPool(), true, + filter.c_str(), filter.length(), + "\\", 1)); } } catch (const Firebird::Exception&) @@ -2571,18 +2567,12 @@ Firebird::string BurpGlobals::toSystem(const Firebird::PathName& from) bool BurpGlobals::skipRelation(const char* name) { if (gbl_sw_meta) - { return true; - } if (!skipDataMatcher) - { return false; - } - skipDataMatcher->reset(); - skipDataMatcher->process(reinterpret_cast(name), static_cast(strlen(name))); - return skipDataMatcher->result(); + return skipDataMatcher->matches(name, strlen(name)); } void BurpGlobals::read_stats(SINT64* stats) @@ -2703,39 +2693,6 @@ void BurpGlobals::print_stats_header() burp_output(false, "\n"); } -UnicodeCollationHolder::UnicodeCollationHolder(MemoryPool& pool) -{ - cs = FB_NEW_POOL(pool) charset; - tt = FB_NEW_POOL(pool) texttype; - - Firebird::IntlUtil::initUtf8Charset(cs); - - Firebird::string collAttributes("ICU-VERSION="); - collAttributes += Jrd::UnicodeUtil::getDefaultIcuVersion(); - Firebird::IntlUtil::setupIcuAttributes(cs, collAttributes, "", collAttributes); - - Firebird::UCharBuffer collAttributesBuffer; - collAttributesBuffer.push(reinterpret_cast(collAttributes.c_str()), - collAttributes.length()); - - if (!Firebird::IntlUtil::initUnicodeCollation(tt, cs, "UNICODE", 0, collAttributesBuffer, Firebird::string())) - Firebird::fatal_exception::raiseFmt("cannot initialize UNICODE collation to use in gbak"); - - charSet = Jrd::CharSet::createInstance(pool, 0, cs); - textType = FB_NEW_POOL(pool) Jrd::TextType(0, tt, charSet); -} - -UnicodeCollationHolder::~UnicodeCollationHolder() -{ - fb_assert(tt->texttype_fn_destroy); - - if (tt->texttype_fn_destroy) - tt->texttype_fn_destroy(tt); - - // cs should be deleted by texttype_fn_destroy call above - delete tt; -} - void BURP_makeSymbol(BurpGlobals* tdgbl, Firebird::string& name) // add double quotes to string { if (tdgbl->gbl_dialect < SQL_DIALECT_V6) diff --git a/src/burp/burp.h b/src/burp/burp.h index 20b77a5afb..53947dde1b 100644 --- a/src/burp/burp.h +++ b/src/burp/burp.h @@ -42,7 +42,7 @@ #include "../common/classes/array.h" #include "../common/classes/fb_pair.h" #include "../common/classes/MetaName.h" -#include "../jrd/SimilarToMatcher.h" +#include "../common/SimilarToRegex.h" #include "../common/status.h" #include "../common/sha.h" #include "../common/classes/ImplementHelper.h" @@ -894,26 +894,6 @@ static const char HDR_SPLIT_TAG6[] = "InterBase/gbak, "; const FB_UINT64 MIN_SPLIT_SIZE = FB_CONST64(2048); // bytes -// Copy&paste from TraceUnicodeUtils.h - fixme !!!!!!!! -class UnicodeCollationHolder -{ -private: - charset* cs; - texttype* tt; - Firebird::AutoPtr charSet; - Firebird::AutoPtr textType; - -public: - explicit UnicodeCollationHolder(Firebird::MemoryPool& pool); - ~UnicodeCollationHolder(); - - Jrd::TextType* getTextType() - { - return textType; - } -}; - - // Global switches and data struct BurpCrypt; @@ -1174,8 +1154,7 @@ public: bool flag_on_line; // indicates whether we will bring the database on-line bool firstMap; // this is the first time we entered get_mapping() bool stdIoMode; // stdin or stdout is used as backup file - Firebird::AutoPtr unicodeCollation; - Firebird::AutoPtr > > skipDataMatcher; + Firebird::AutoPtr skipDataMatcher; public: Firebird::string toSystem(const Firebird::PathName& from); diff --git a/src/common/SimilarToRegex.cpp b/src/common/SimilarToRegex.cpp new file mode 100644 index 0000000000..1d49478918 --- /dev/null +++ b/src/common/SimilarToRegex.cpp @@ -0,0 +1,821 @@ +/* + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Adriano dos Santos Fernandes + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2019 Adriano dos Santos Fernandes + * and all contributors signed below. + * + */ + +#include "firebird.h" +#include "../common/SimilarToRegex.h" +#include "../common/StatusArg.h" +#include + +using namespace Firebird; + +namespace +{ + static const unsigned FLAG_PREFER_FEWER = 0x01; + static const unsigned FLAG_CASE_INSENSITIVE = 0x02; + static const unsigned FLAG_GROUP_CAPTURE = 0x04; + + //// TODO: Verify usage of U8_NEXT_UNSAFE. + class SimilarToCompiler + { + public: + SimilarToCompiler(MemoryPool& pool, AutoPtr& regexp, unsigned aFlags, + const char* aPatternStr, unsigned aPatternLen, + const char* escapeStr, unsigned escapeLen) + : re2PatternStr(pool), + patternStr(aPatternStr), + patternPos(0), + patternLen(aPatternLen), + flags(aFlags), + useEscape(escapeStr != nullptr) + { + if (escapeStr) + { + int32_t escapePos = 0; + U8_NEXT_UNSAFE(escapeStr, escapePos, escapeChar); + + if (escapePos != escapeLen) + status_exception::raise(Arg::Gds(isc_escape_invalid)); + } + + if (flags & FLAG_CASE_INSENSITIVE) + re2PatternStr.append("(?i)"); + + if (flags & FLAG_GROUP_CAPTURE) + re2PatternStr.append("("); + + int parseFlags; + parseExpr(&parseFlags); + + if (flags & FLAG_GROUP_CAPTURE) + re2PatternStr.append(")"); + + // Check for proper termination. + if (patternPos < patternLen) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + + RE2::Options options; + options.set_log_errors(false); + options.set_dot_nl(true); + + re2::StringPiece sp((const char*) re2PatternStr.c_str(), re2PatternStr.length()); + regexp = FB_NEW_POOL(pool) RE2(sp, options); + + if (!regexp->ok()) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + } + + bool hasChar() + { + return patternPos < patternLen; + } + + UChar32 getChar() + { + fb_assert(hasChar()); + UChar32 c; + U8_NEXT_UNSAFE(patternStr, patternPos, c); + return c; + } + + UChar32 peekChar() + { + auto savePos = patternPos; + auto c = getChar(); + patternPos = savePos; + return c; + } + + bool isRep(UChar32 c) const + { + return c == '*' || c == '+' || c == '?' || c == '{'; + } + + bool isSpecial(UChar32 c) + { + switch (c) + { + case '^': + case '-': + case '_': + case '%': + case '[': + case ']': + case '(': + case ')': + case '{': + case '}': + case '|': + case '?': + case '+': + case '*': + return true; + + default: + return false; + } + } + + bool isRe2Special(UChar32 c) + { + switch (c) + { + case '\\': + case '$': + case '.': + case '^': + case '-': + case '_': + case '[': + case ']': + case '(': + case ')': + case '{': + case '}': + case '|': + case '?': + case '+': + case '*': + return true; + + default: + return false; + } + } + + void parseExpr(int* parseFlagOut) + { + while (true) + { + int parseFlags; + parseTerm(&parseFlags); + *parseFlagOut &= ~(~parseFlags & PARSE_FLAG_NOT_EMPTY); + *parseFlagOut |= parseFlags; + + auto savePos = patternPos; + UChar32 c; + + if (!hasChar() || (c = getChar()) != '|') + { + patternPos = savePos; + break; + } + + re2PatternStr.append("|"); + } + } + + void parseTerm(int* parseFlagOut) + { + *parseFlagOut = 0; + + bool first = true; + + while (hasChar()) + { + auto c = peekChar(); + + if (c != '|' && c != ')') + { + int parseFlags; + parseFactor(&parseFlags); + + *parseFlagOut |= parseFlags & PARSE_FLAG_NOT_EMPTY; + + if (first) + { + *parseFlagOut |= parseFlags; + first = false; + } + } + else + break; + } + } + + void parseFactor(int* parseFlagOut) + { + int parseFlags; + parsePrimary(&parseFlags); + + UChar32 op; + + if (!hasChar() || !isRep((op = peekChar()))) + { + *parseFlagOut = parseFlags; + return; + } + + if (!(parseFlags & PARSE_FLAG_NOT_EMPTY) && op != '?') + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + + fb_assert(op == '*' || op == '+' || op == '?' || op == '{'); + + if (op == '*') + { + re2PatternStr.append((flags & FLAG_PREFER_FEWER) ? "*?" : "*"); + *parseFlagOut = 0; + ++patternPos; + } + else if (op == '+') + { + re2PatternStr.append((flags & FLAG_PREFER_FEWER) ? "+?" : "+"); + *parseFlagOut = PARSE_FLAG_NOT_EMPTY; + ++patternPos; + } + else if (op == '?') + { + re2PatternStr.append((flags & FLAG_PREFER_FEWER) ? "??" : "?"); + *parseFlagOut = 0; + ++patternPos; + } + else if (op == '{') + { + const auto repeatStart = patternPos++; + + bool comma = false; + string s1, s2; + + while (true) + { + if (!hasChar()) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + + UChar32 c = getChar(); + + if (c == '}') + { + if (s1.isEmpty()) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + break; + } + else if (c == ',') + { + if (comma) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + comma = true; + } + else + { + if (c >= '0' && c <= '9') + { + if (comma) + s2 += (char) c; + else + s1 += (char) c; + } + else + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + } + } + + const int n1 = atoi(s1.c_str()); + *parseFlagOut = n1 == 0 ? 0 : PARSE_FLAG_NOT_EMPTY; + + re2PatternStr.append(patternStr + repeatStart, patternStr + patternPos); + + if (flags & FLAG_PREFER_FEWER) + re2PatternStr.append("?"); + } + + if (hasChar() && isRep(peekChar())) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + } + + void parsePrimary(int* parseFlagOut) + { + *parseFlagOut = 0; + + fb_assert(hasChar()); + auto savePos = patternPos; + auto op = getChar(); + + if (op == '_') + { + *parseFlagOut |= PARSE_FLAG_NOT_EMPTY; + re2PatternStr.append("."); + return; + } + else if (op == '%') + { + re2PatternStr.append((flags & FLAG_PREFER_FEWER) ? ".*?" : ".*"); + return; + } + else if (op == '[') + { + struct + { + const char* similarClass; + const char* re2ClassInclude; + const char* re2ClassExclude; + } static const classes[] = + { + {"alnum", "[:alnum:]", "[:^alnum:]"}, + {"alpha", "[:alpha:]", "[:^alpha:]"}, + {"digit", "[:digit:]", "[:^digit:]"}, + {"lower", "[:lower:]", "[:^lower:]"}, + {"space", " ", "\\x00-\\x1F\\x21-\\x{10FFFF}"}, + {"upper", "[:upper:]", "[:^upper:]"}, + {"whitespace", "[:space:]", "[:^space:]"} + }; + + struct Item + { + int clazz; + unsigned firstStart, firstEnd, lastStart, lastEnd; + }; + Array items; + unsigned includeCount = 0; + bool exclude = false; + + do + { + if (!hasChar()) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + + unsigned charSavePos = patternPos; + UChar32 c = getChar(); + bool range = false; + bool charClass = false; + + if (useEscape && c == escapeChar) + { + if (!hasChar()) + status_exception::raise(Arg::Gds(isc_escape_invalid)); + + charSavePos = patternPos; + c = getChar(); + + if (!(c == escapeChar || isSpecial(c))) + status_exception::raise(Arg::Gds(isc_escape_invalid)); + } + else + { + if (c == '[') + charClass = true; + else if (c == '^') + { + if (exclude) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + + exclude = true; + continue; + } + } + + Item item; + + if (!exclude) + ++includeCount; + + if (charClass) + { + if (!hasChar() || getChar() != ':') + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + + charSavePos = patternPos; + + while (hasChar() && getChar() != ':') + ; + + const SLONG len = patternPos - charSavePos - 1; + + if (!hasChar() || getChar() != ']') + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + + for (item.clazz = 0; item.clazz < FB_NELEM(classes); ++item.clazz) + { + if (fb_utils::strnicmp(patternStr + charSavePos, + classes[item.clazz].similarClass, len) == 0) + { + break; + } + } + + if (item.clazz >= FB_NELEM(classes)) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + } + else + { + item.clazz = -1; + + item.firstStart = item.lastStart = charSavePos; + item.firstEnd = item.lastEnd = patternPos; + + if (hasChar() && peekChar() == '-') + { + getChar(); + + charSavePos = patternPos; + c = getChar(); + + if (useEscape && c == escapeChar) + { + if (!hasChar()) + status_exception::raise(Arg::Gds(isc_escape_invalid)); + + charSavePos = patternPos; + c = getChar(); + + if (!(c == escapeChar || isSpecial(c))) + status_exception::raise(Arg::Gds(isc_escape_invalid)); + } + + item.lastStart = charSavePos; + item.lastEnd = patternPos; + } + } + + items.add(item); + + if (!hasChar()) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + } while (peekChar() != ']'); + + auto appendItem = [&](const Item& item, bool negated) { + if (item.clazz != -1) + { + re2PatternStr.append(negated ? + classes[item.clazz].re2ClassExclude : + classes[item.clazz].re2ClassInclude); + } + else + { + if (negated) + { + UChar32 c; + char hex[20]; + + int32_t cPos = item.firstStart; + U8_NEXT_UNSAFE(patternStr, cPos, c); + + if (c > 0) + { + re2PatternStr.append("\\x00"); + re2PatternStr.append("-"); + + sprintf(hex, "\\x{%X}", (int) c - 1); + re2PatternStr.append(hex); + } + + cPos = item.lastStart; + U8_NEXT_UNSAFE(patternStr, cPos, c); + + if (c < 0x10FFFF) + { + sprintf(hex, "\\x{%X}", (int) c + 1); + re2PatternStr.append(hex); + re2PatternStr.append("-"); + re2PatternStr.append("\\x{10FFFF}"); + } + } + else + { + if (isRe2Special(patternStr[item.firstStart])) + re2PatternStr.append("\\"); + + re2PatternStr.append(patternStr + item.firstStart, patternStr + item.firstEnd); + + if (item.lastStart != item.firstStart) + { + re2PatternStr.append("-"); + + if (isRe2Special(patternStr[item.lastStart])) + re2PatternStr.append("\\"); + + re2PatternStr.append(patternStr + item.lastStart, patternStr + item.lastEnd); + } + } + } + }; + + if (exclude && includeCount > 1) + { + re2PatternStr.append("(?:"); + + for (unsigned i = 0; i < includeCount; ++i) + { + if (i != 0) + re2PatternStr.append("|"); + + re2PatternStr.append("["); + re2PatternStr.append("^"); + appendItem(items[i], true); + + for (unsigned j = includeCount; j < items.getCount(); ++j) + appendItem(items[j], false); + + re2PatternStr.append("]"); + } + + re2PatternStr.append(")"); + } + else + { + re2PatternStr.append("["); + + if (exclude) + re2PatternStr.append("^"); + + for (unsigned i = 0; i < items.getCount(); ++i) + appendItem(items[i], exclude && i < includeCount); + + re2PatternStr.append("]"); + } + + getChar(); + *parseFlagOut |= PARSE_FLAG_NOT_EMPTY; + } + else if (op == '(') + { + re2PatternStr.append(flags & FLAG_GROUP_CAPTURE ? "(" : "(?:"); + + int parseFlags; + parseExpr(&parseFlags); + + if (!hasChar() || getChar() != ')') + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + + re2PatternStr.append(")"); + + *parseFlagOut |= parseFlags & PARSE_FLAG_NOT_EMPTY; + } + else + { + patternPos = savePos; + + bool controlChar = false; + + do + { + auto charSavePos = patternPos; + op = getChar(); + + if (useEscape && op == escapeChar) + { + charSavePos = patternPos; + op = getChar(); + + if (!isSpecial(op) && op != escapeChar) + status_exception::raise(Arg::Gds(isc_escape_invalid)); + } + else + { + if (isSpecial(op)) + { + controlChar = true; + patternPos = charSavePos; + } + } + + if (!controlChar) + { + if (isRe2Special(op)) + re2PatternStr.append("\\"); + + re2PatternStr.append(patternStr + charSavePos, patternStr + patternPos); + } + } while (!controlChar && hasChar()); + + if (patternPos == savePos) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + + *parseFlagOut |= PARSE_FLAG_NOT_EMPTY; + } + } + + const string& getRe2PatternStr() const + { + return re2PatternStr; + } + + private: + static const int PARSE_FLAG_NOT_EMPTY = 1; // known never to match empty string + + string re2PatternStr; + const char* patternStr; + int32_t patternPos; + int32_t patternLen; + UChar32 escapeChar; + unsigned flags; + bool useEscape; + }; + + class SubstringSimilarCompiler + { + public: + SubstringSimilarCompiler(MemoryPool& pool, AutoPtr& regexp, unsigned flags, + const char* aPatternStr, unsigned aPatternLen, + const char* escapeStr, unsigned escapeLen) + : patternStr(aPatternStr), + patternPos(0), + patternLen(aPatternLen) + { + int32_t escapePos = 0; + U8_NEXT_UNSAFE(escapeStr, escapePos, escapeChar); + + if (escapePos != escapeLen) + status_exception::raise(Arg::Gds(isc_escape_invalid)); + + unsigned positions[2]; + unsigned part = 0; + + while (hasChar()) + { + auto c = getChar(); + + if (c != escapeChar) + continue; + + if (!hasChar()) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + + c = getChar(); + + if (c == '"') + { + if (part >= 2) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + + positions[part++] = patternPos; + } + } + + if (part != 2) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + + AutoPtr regexp1, regexp2, regexp3; + + SimilarToCompiler compiler1(pool, regexp1, (flags & FLAG_CASE_INSENSITIVE) | FLAG_PREFER_FEWER, + aPatternStr, positions[0] - escapeLen - 1, escapeStr, escapeLen); + + SimilarToCompiler compiler2(pool, regexp2, (flags & FLAG_CASE_INSENSITIVE), + aPatternStr + positions[0], positions[1] - positions[0] - escapeLen - 1, escapeStr, escapeLen); + + SimilarToCompiler compiler3(pool, regexp3, (flags & FLAG_CASE_INSENSITIVE) | FLAG_PREFER_FEWER, + aPatternStr + positions[1], patternLen - positions[1], escapeStr, escapeLen); + + string finalRe2Pattern; + finalRe2Pattern.reserve( + 1 + // ( + compiler1.getRe2PatternStr().length() + + 2 + // )( + compiler2.getRe2PatternStr().length() + + 2 + // )( + compiler3.getRe2PatternStr().length() + + 1 // ) + ); + + finalRe2Pattern.append("("); + finalRe2Pattern.append(compiler1.getRe2PatternStr()); + finalRe2Pattern.append(")("); + finalRe2Pattern.append(compiler2.getRe2PatternStr()); + finalRe2Pattern.append(")("); + finalRe2Pattern.append(compiler3.getRe2PatternStr()); + finalRe2Pattern.append(")"); + + RE2::Options options; + options.set_log_errors(false); + options.set_dot_nl(true); + + re2::StringPiece sp((const char*) finalRe2Pattern.c_str(), finalRe2Pattern.length()); + regexp = FB_NEW_POOL(pool) RE2(sp, options); + + if (!regexp->ok()) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + } + + bool hasChar() + { + return patternPos < patternLen; + } + + UChar32 getChar() + { + fb_assert(hasChar()); + UChar32 c; + U8_NEXT_UNSAFE(patternStr, patternPos, c); + return c; + } + + UChar32 peekChar() + { + auto savePos = patternPos; + auto c = getChar(); + patternPos = savePos; + return c; + } + + private: + const char* patternStr; + int32_t patternPos; + int32_t patternLen; + UChar32 escapeChar; + }; +} // namespace + +namespace Firebird { + + +SimilarToRegex::SimilarToRegex(MemoryPool& pool, bool caseInsensitive, + const char* patternStr, unsigned patternLen, const char* escapeStr, unsigned escapeLen) + : PermanentStorage(pool) +{ + SimilarToCompiler compiler(pool, regexp, + FLAG_GROUP_CAPTURE | FLAG_PREFER_FEWER | (caseInsensitive ? FLAG_CASE_INSENSITIVE : 0), + patternStr, patternLen, escapeStr, escapeLen); +} + +bool SimilarToRegex::matches(const char* buffer, unsigned bufferLen, Array* matchPosArray) +{ + re2::StringPiece sp(buffer, bufferLen); + + if (matchPosArray) + { + const int argsCount = regexp->NumberOfCapturingGroups(); + + Array resSps(argsCount); + resSps.resize(argsCount); + + Array args(argsCount); + args.resize(argsCount); + + Array argsPtr(argsCount); + + { // scope + auto resSp = resSps.begin(); + + for (auto& arg : args) + { + arg = resSp++; + argsPtr.push(&arg); + } + } + + if (RE2::FullMatchN(sp, *regexp.get(), argsPtr.begin(), argsCount)) + { + matchPosArray->clear(); + + for (const auto resSp : resSps) + { + matchPosArray->push(MatchPos{ + static_cast(resSp.data() - sp.begin()), + static_cast(resSp.length()) + }); + } + + return true; + } + else + return false; + } + else + return RE2::FullMatch(sp, *regexp.get()); +} + +//--------------------- + +SubstringSimilarRegex::SubstringSimilarRegex(MemoryPool& pool, bool caseInsensitive, + const char* patternStr, unsigned patternLen, const char* escapeStr, unsigned escapeLen) + : PermanentStorage(pool) +{ + SubstringSimilarCompiler compiler(pool, regexp, + (caseInsensitive ? FLAG_CASE_INSENSITIVE : 0), + patternStr, patternLen, escapeStr, escapeLen); +} + +bool SubstringSimilarRegex::matches(const char* buffer, unsigned bufferLen, + unsigned* resultStart, unsigned* resultLength) +{ + re2::StringPiece sp(buffer, bufferLen); + + re2::StringPiece spResult; + + if (RE2::FullMatch(sp, *regexp.get(), nullptr, &spResult, nullptr)) + { + *resultStart = spResult.begin() - buffer; + *resultLength = spResult.length(); + return true; + } + else + return false; +} + + +} // namespace Firebird diff --git a/src/common/SimilarToRegex.h b/src/common/SimilarToRegex.h new file mode 100644 index 0000000000..e1b2554bb9 --- /dev/null +++ b/src/common/SimilarToRegex.h @@ -0,0 +1,75 @@ +/* + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Adriano dos Santos Fernandes + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2019 Adriano dos Santos Fernandes + * and all contributors signed below. + * + */ + +#ifndef COMMON_SIMILAR_TO_REGEX_H +#define COMMON_SIMILAR_TO_REGEX_H + +#include "firebird.h" +#include +#include "../common/classes/auto.h" +#include "../common/classes/array.h" +#include "../common/classes/fb_string.h" + +namespace Firebird { + + +//// FIXME: Leak re2::RE2 when destroyed by pool. +class SimilarToRegex : public PermanentStorage +{ +public: + struct MatchPos + { + unsigned start; + unsigned length; + }; + +public: + SimilarToRegex(MemoryPool& pool, bool caseInsensitive, + const char* patternStr, unsigned patternLen, const char* escapeStr, unsigned escapeLen); + +public: + bool matches(const char* buffer, unsigned bufferLen, Array* matchPosArray = nullptr); + +private: + AutoPtr regexp; +}; + +//// FIXME: Leak re2::RE2 when destroyed by pool. +// Given a regular expression R1#R2#R3 and the string S: +// - Find the shortest substring of S that matches R1 while the remainder (S23) matches R2R3; +// - Find the longest (S2) substring of S23 that matches R2 while the remainder matches R3; +// - Return S2. +class SubstringSimilarRegex : public PermanentStorage +{ +public: + SubstringSimilarRegex(MemoryPool& pool, bool caseInsensitive, + const char* patternStr, unsigned patternLen, const char* escapeStr, unsigned escapeLen); + +public: + bool matches(const char* buffer, unsigned bufferLen, unsigned* resultStart, unsigned* resultLength); + +private: + AutoPtr regexp; +}; + + +} // namespace Firebird + +#endif // COMMON_SIMILAR_TO_REGEX_H diff --git a/src/common/TextType.cpp b/src/common/TextType.cpp index ff48778f39..cd69aa5c80 100644 --- a/src/common/TextType.cpp +++ b/src/common/TextType.cpp @@ -169,33 +169,6 @@ TextType::TextType(TTYPE_ID _type, texttype *_tt, CharSet* _cs) memset(&canonicalChars[conversions[i].ch], 0, sizeof(ULONG)); } } - - struct Conversion2 - { - const char* str; - UCHAR* buffer; - }; - - const Conversion2 conversions2[] = - { - {"0123456789", reinterpret_cast(canonicalNumbers)}, - {"abcdefghijklmnopqrstuvwxyz", reinterpret_cast(canonicalLowerLetters)}, - {"ABCDEFGHIJKLMNOPQRSTUVWXYZ", reinterpret_cast(canonicalUpperLetters)}, - {" \t\v\r\n\f", reinterpret_cast(canonicalWhiteSpaces)} - }; - - for (int i = 0; i < FB_NELEM(conversions2); i++) - { - UCHAR temp[sizeof(ULONG)]; - - for (const char* p = conversions2[i].str; *p; ++p) - { - USHORT code = static_cast(*p); - ULONG length = getCharSet()->getConvFromUnicode().convert(sizeof(code), &code, sizeof(temp), temp); - const size_t pos = (p - conversions2[i].str) * getCanonicalWidth(); - canonical(length, temp, sizeof(ULONG), &conversions2[i].buffer[pos]); - } - } } diff --git a/src/common/TextType.h b/src/common/TextType.h index 469afb530b..bfe0e68a26 100644 --- a/src/common/TextType.h +++ b/src/common/TextType.h @@ -138,47 +138,8 @@ public: return reinterpret_cast(&canonicalChars[ch]); } - const UCHAR* getCanonicalNumbers(int* count = NULL) const - { - if (count) - *count = 10; - return reinterpret_cast(canonicalNumbers); - } - - const UCHAR* getCanonicalLowerLetters(int* count = NULL) const - { - if (count) - *count = 26; - return reinterpret_cast(canonicalLowerLetters); - } - - const UCHAR* getCanonicalUpperLetters(int* count = NULL) const - { - if (count) - *count = 26; - return reinterpret_cast(canonicalUpperLetters); - } - - const UCHAR* getCanonicalWhiteSpaces(int* count = NULL) const - { - if (count) - *count = 6; - return reinterpret_cast(canonicalWhiteSpaces); - } - - const UCHAR* getCanonicalSpace(int* count = NULL) const - { - if (count) - *count = 1; - return getCanonicalChar(CHAR_SPACE); - } - private: ULONG canonicalChars[CHAR_COUNT]; - ULONG canonicalNumbers[10]; - ULONG canonicalLowerLetters[26]; - ULONG canonicalUpperLetters[26]; - ULONG canonicalWhiteSpaces[6]; }; } // namespace Jrd diff --git a/src/common/unicode_util.cpp b/src/common/unicode_util.cpp index cf51d5b5cd..af5b3ff98f 100644 --- a/src/common/unicode_util.cpp +++ b/src/common/unicode_util.cpp @@ -1031,6 +1031,37 @@ INTL_BOOL UnicodeUtil::utf32WellFormed(ULONG len, const ULONG* str, ULONG* offen return true; // well-formed } +void UnicodeUtil::utf8Normalize(UCharBuffer& data) +{ + ICU* icu = loadICU("", ""); + + HalfStaticArray utf16Buffer(data.getCount()); + USHORT errCode; + ULONG errPosition; + ULONG utf16BufferLen = utf8ToUtf16(data.getCount(), data.begin(), data.getCount() * sizeof(USHORT), + utf16Buffer.getBuffer(data.getCount()), &errCode, &errPosition); + + UTransliterator* trans = icu->getCiAiTransliterator(); + + if (trans) + { + const int32_t capacity = utf16Buffer.getCount() * sizeof(USHORT); + int32_t len = utf16BufferLen / sizeof(USHORT); + int32_t limit = len; + + UErrorCode errorCode = U_ZERO_ERROR; + icu->utransTransUChars(trans, reinterpret_cast(utf16Buffer.begin()), + &len, capacity, 0, &limit, &errorCode); + icu->releaseCiAiTransliterator(trans); + + len = utf16ToUtf8(utf16BufferLen, utf16Buffer.begin(), + len * 4, data.getBuffer(len * 4, false), + &errCode, &errPosition); + + data.shrink(len); + } +} + UnicodeUtil::ICU* UnicodeUtil::loadICU(const string& icuVersion, const string& configInfo) { ObjectsArray versions; diff --git a/src/common/unicode_util.h b/src/common/unicode_util.h index ff2e30ef27..e57e9a9eed 100644 --- a/src/common/unicode_util.h +++ b/src/common/unicode_util.h @@ -177,6 +177,8 @@ public: static INTL_BOOL utf16WellFormed(ULONG len, const USHORT* str, ULONG* offending_position); static INTL_BOOL utf32WellFormed(ULONG len, const ULONG* str, ULONG* offending_position); + static void utf8Normalize(Firebird::UCharBuffer& data); + static ConversionICU& getConversionICU(); static ICU* loadICU(const Firebird::string& icuVersion, const Firebird::string& configInfo); static bool getCollVersion(const Firebird::string& icuVersion, diff --git a/src/dsql/BoolNodes.cpp b/src/dsql/BoolNodes.cpp index 205fc11cc2..5b6e16a9d1 100644 --- a/src/dsql/BoolNodes.cpp +++ b/src/dsql/BoolNodes.cpp @@ -945,7 +945,7 @@ bool ComparativeBoolNode::stringBoolean(thread_db* tdbb, jrd_req* request, dsc* else // nod_similar { impure->vlu_misc.vlu_invariant = evaluator = obj->createSimilarToMatcher( - *tdbb->getDefaultPool(), p2, l2, escape_str, escape_length); + tdbb, *tdbb->getDefaultPool(), p2, l2, escape_str, escape_length); } } else @@ -961,7 +961,7 @@ bool ComparativeBoolNode::stringBoolean(thread_db* tdbb, jrd_req* request, dsc* } else // nod_similar { - evaluator = obj->createSimilarToMatcher(*tdbb->getDefaultPool(), + evaluator = obj->createSimilarToMatcher(tdbb, *tdbb->getDefaultPool(), p2, l2, escape_str, escape_length); } @@ -1152,7 +1152,7 @@ bool ComparativeBoolNode::stringFunction(thread_db* tdbb, jrd_req* request, else // nod_similar { impure->vlu_misc.vlu_invariant = evaluator = obj->createSimilarToMatcher( - *tdbb->getDefaultPool(), p2, l2, escape_str, escape_length); + tdbb, *tdbb->getDefaultPool(), p2, l2, escape_str, escape_length); } } else @@ -1170,7 +1170,7 @@ bool ComparativeBoolNode::stringFunction(thread_db* tdbb, jrd_req* request, return obj->like(*tdbb->getDefaultPool(), p1, l1, p2, l2, escape_str, escape_length); // nod_similar - return obj->similarTo(*tdbb->getDefaultPool(), p1, l1, p2, l2, escape_str, escape_length); + return obj->similarTo(tdbb, *tdbb->getDefaultPool(), p1, l1, p2, l2, escape_str, escape_length); } // Handle MATCHES diff --git a/src/dsql/ExprNodes.cpp b/src/dsql/ExprNodes.cpp index b2208674f1..dd75a5d6b7 100644 --- a/src/dsql/ExprNodes.cpp +++ b/src/dsql/ExprNodes.cpp @@ -11873,7 +11873,7 @@ dsc* SubstringSimilarNode::execute(thread_db* tdbb, jrd_req* request) const delete impure->vlu_misc.vlu_invariant; impure->vlu_misc.vlu_invariant = evaluator = collation->createSubstringSimilarMatcher( - *tdbb->getDefaultPool(), patternStr, patternLen, escapeStr, escapeLen); + tdbb, *tdbb->getDefaultPool(), patternStr, patternLen, escapeStr, escapeLen); impure->vlu_flags |= VLU_computed; } @@ -11885,7 +11885,7 @@ dsc* SubstringSimilarNode::execute(thread_db* tdbb, jrd_req* request) const } else { - autoEvaluator = evaluator = collation->createSubstringSimilarMatcher(*tdbb->getDefaultPool(), + autoEvaluator = evaluator = collation->createSubstringSimilarMatcher(tdbb, *tdbb->getDefaultPool(), patternStr, patternLen, escapeStr, escapeLen); } diff --git a/src/jrd/Collation.cpp b/src/jrd/Collation.cpp index 46e56d887e..f7d2445163 100644 --- a/src/jrd/Collation.cpp +++ b/src/jrd/Collation.cpp @@ -99,16 +99,177 @@ #include "../jrd/intl_classes.h" #include "../jrd/lck_proto.h" #include "../jrd/intl_classes.h" +#include "../jrd/intl_proto.h" #include "../jrd/Collation.h" #include "../common/TextType.h" +#include "../common/SimilarToRegex.h" -#include "../jrd/SimilarToMatcher.h" - +using namespace Firebird; using namespace Jrd; namespace { +//// TODO: NONE / OCTETS. +class Re2SimilarMatcher : public PatternMatcher +{ +public: + Re2SimilarMatcher(thread_db* tdbb, MemoryPool& pool, TextType* textType, + const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen) + : PatternMatcher(pool, textType), + buffer(pool) + { + CsConvert converter = INTL_convert_lookup(tdbb, CS_UTF8, textType->getCharSet()->getId()); + + UCharBuffer patternBuffer, escapeBuffer; + + converter.convert(patternLen, patternStr, patternBuffer); + + if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + UnicodeUtil::utf8Normalize(patternBuffer); + + if (escapeStr) + { + converter.convert(escapeLen, escapeStr, escapeBuffer); + + if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + UnicodeUtil::utf8Normalize(escapeBuffer); + } + + regex = FB_NEW_POOL(pool) SimilarToRegex(pool, + (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE), + (const char*) patternBuffer.begin(), patternBuffer.getCount(), + (escapeStr ? (const char*) escapeBuffer.begin() : nullptr), escapeBuffer.getCount()); + } + +public: + static Re2SimilarMatcher* create(thread_db* tdbb, MemoryPool& pool, TextType* textType, + const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen) + { + return FB_NEW_POOL(pool) Re2SimilarMatcher(tdbb, pool, textType, patternStr, patternLen, escapeStr, escapeLen); + } + + static bool evaluate(thread_db* tdbb, MemoryPool& pool, TextType* textType, const UCHAR* str, SLONG strLen, + const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen) + { + Re2SimilarMatcher matcher(tdbb, pool, textType, patternStr, patternLen, escapeStr, escapeLen); + matcher.process(str, strLen); + return matcher.result(); + } + +public: + virtual void reset() + { + buffer.shrink(0); + } + + virtual bool process(const UCHAR* data, SLONG dataLen) + { + const FB_SIZE_T pos = buffer.getCount(); + memcpy(buffer.getBuffer(pos + dataLen) + pos, data, dataLen); + return true; + } + + virtual bool result() + { + if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + UnicodeUtil::utf8Normalize(buffer); + + return regex->matches((const char*) buffer.begin(), buffer.getCount()); + } + +private: + AutoPtr regex; + UCharBuffer buffer; +}; + +class Re2SubstringSimilarMatcher : public BaseSubstringSimilarMatcher +{ +public: + Re2SubstringSimilarMatcher(thread_db* tdbb, MemoryPool& pool, TextType* textType, + const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen) + : BaseSubstringSimilarMatcher(pool, textType), + buffer(pool), + resultStart(0), + resultLength(0) + { + CsConvert converter = INTL_convert_lookup(tdbb, textType->getCharSet()->getId(), CS_UTF8); + + UCharBuffer patternBuffer, escapeBuffer; + + converter.convert(patternLen, patternStr, patternBuffer); + + if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + UnicodeUtil::utf8Normalize(patternBuffer); + + if (escapeStr) + { + converter.convert(escapeLen, escapeStr, escapeBuffer); + + if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + UnicodeUtil::utf8Normalize(escapeBuffer); + } + + regex = FB_NEW_POOL(pool) SubstringSimilarRegex(pool, + (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE), + (const char*) patternBuffer.begin(), patternBuffer.getCount(), + (escapeStr ? (const char*) escapeBuffer.begin() : nullptr), escapeBuffer.getCount()); + } + + virtual ~Re2SubstringSimilarMatcher() + { + } + +public: + static Re2SubstringSimilarMatcher* create(thread_db* tdbb, MemoryPool& pool, TextType* textType, + const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen) + { + return FB_NEW_POOL(pool) Re2SubstringSimilarMatcher(tdbb, pool, textType, + patternStr, patternLen, escapeStr, escapeLen); + } + + static bool evaluate(thread_db* tdbb, MemoryPool& pool, TextType* textType, const UCHAR* str, SLONG strLen, + const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen) + { + Re2SubstringSimilarMatcher matcher(tdbb, pool, textType, patternStr, patternLen, escapeStr, escapeLen); + matcher.process(str, strLen); + return matcher.result(); + } + +public: + virtual void reset() + { + buffer.shrink(0); + resultStart = resultLength = 0; + } + + virtual bool process(const UCHAR* data, SLONG dataLen) + { + const FB_SIZE_T pos = buffer.getCount(); + memcpy(buffer.getBuffer(pos + dataLen) + pos, data, dataLen); + return true; + } + + virtual bool result() + { + if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + UnicodeUtil::utf8Normalize(buffer); + + return regex->matches((const char*) buffer.begin(), buffer.getCount(), &resultStart, &resultLength); + } + + virtual void getResultInfo(unsigned* start, unsigned* length) + { + *start = resultStart; + *length = resultLength; + } + +private: + AutoPtr regex; + UCharBuffer buffer; + unsigned resultStart, resultLength; +}; + // constants used in matches and sleuth const int CHAR_GDML_MATCH_ONE = TextType::CHAR_QUESTION_MARK; const int CHAR_GDML_MATCH_ANY = TextType::CHAR_ASTERISK; @@ -725,8 +886,6 @@ template < typename pStartsMatcher, typename pContainsMatcher, typename pLikeMatcher, - typename pSimilarToMatcher, - typename pSubstringSimilarMatcher, typename pMatchesMatcher, typename pSleuthMatcher > @@ -781,22 +940,22 @@ public: getCharSet()->getSqlMatchOne(), getCharSet()->getSqlMatchOneLength()); } - virtual bool similarTo(MemoryPool& pool, const UCHAR* s, SLONG sl, + virtual bool similarTo(thread_db* tdbb, MemoryPool& pool, const UCHAR* s, SLONG sl, const UCHAR* p, SLONG pl, const UCHAR* escape, SLONG escapeLen) { - return pSimilarToMatcher::evaluate(pool, this, s, sl, p, pl, escape, escapeLen); + return Re2SimilarMatcher::evaluate(tdbb, pool, this, s, sl, p, pl, escape, escapeLen); } - virtual PatternMatcher* createSimilarToMatcher(MemoryPool& pool, const UCHAR* p, SLONG pl, + virtual PatternMatcher* createSimilarToMatcher(thread_db* tdbb, MemoryPool& pool, const UCHAR* p, SLONG pl, const UCHAR* escape, SLONG escapeLen) { - return pSimilarToMatcher::create(pool, this, p, pl, escape, escapeLen); + return Re2SimilarMatcher::create(tdbb, pool, this, p, pl, escape, escapeLen); } - virtual BaseSubstringSimilarMatcher* createSubstringSimilarMatcher(MemoryPool& pool, + virtual BaseSubstringSimilarMatcher* createSubstringSimilarMatcher(thread_db* tdbb, MemoryPool& pool, const UCHAR* p, SLONG pl, const UCHAR* escape, SLONG escapeLen) { - return pSubstringSimilarMatcher::create(pool, this, p, pl, escape, escapeLen); + return Re2SubstringSimilarMatcher::create(tdbb, pool, this, p, pl, escape, escapeLen); } virtual bool contains(MemoryPool& pool, const UCHAR* s, SLONG sl, const UCHAR* p, SLONG pl) @@ -823,8 +982,6 @@ Collation* newCollation(MemoryPool& pool, TTYPE_ID id, texttype* tt, CharSet* cs StartsMatcherUCharDirect, ContainsMatcherUCharDirect, LikeMatcher, - SimilarToMatcher, - SubstringSimilarMatcher, MatchesMatcher, SleuthMatcher > DirectImpl; @@ -833,8 +990,6 @@ Collation* newCollation(MemoryPool& pool, TTYPE_ID id, texttype* tt, CharSet* cs StartsMatcherUCharCanonical, ContainsMatcher, LikeMatcher, - SimilarToMatcher, - SubstringSimilarMatcher, MatchesMatcher, SleuthMatcher > NonDirectImpl; diff --git a/src/jrd/Collation.h b/src/jrd/Collation.h index 6607f97c66..268411d5f2 100644 --- a/src/jrd/Collation.h +++ b/src/jrd/Collation.h @@ -66,12 +66,12 @@ public: virtual PatternMatcher* createLikeMatcher(MemoryPool& pool, const UCHAR* p, SLONG pl, const UCHAR* escape, SLONG escapeLen) = 0; - virtual bool similarTo(MemoryPool& pool, const UCHAR* s, SLONG sl, const UCHAR* p, SLONG pl, + virtual bool similarTo(thread_db* tdbb, MemoryPool& pool, const UCHAR* s, SLONG sl, const UCHAR* p, SLONG pl, const UCHAR* escape, SLONG escapeLen) = 0; - virtual PatternMatcher* createSimilarToMatcher(MemoryPool& pool, const UCHAR* p, SLONG pl, + virtual PatternMatcher* createSimilarToMatcher(thread_db* tdbb, MemoryPool& pool, const UCHAR* p, SLONG pl, const UCHAR* escape, SLONG escapeLen) = 0; - virtual BaseSubstringSimilarMatcher* createSubstringSimilarMatcher(MemoryPool& pool, + virtual BaseSubstringSimilarMatcher* createSubstringSimilarMatcher(thread_db* tdbb, MemoryPool& pool, const UCHAR* p, SLONG pl, const UCHAR* escape, SLONG escapeLen) = 0; virtual bool contains(MemoryPool& pool, const UCHAR* s, SLONG sl, const UCHAR* p, SLONG pl) = 0; diff --git a/src/jrd/IntlManager.cpp b/src/jrd/IntlManager.cpp index ccc9857e1d..4e4a517ac1 100644 --- a/src/jrd/IntlManager.cpp +++ b/src/jrd/IntlManager.cpp @@ -654,6 +654,7 @@ bool IntlManager::lookupCollation(const string& collationName, attributes, specificAttributes, specificAttributesLen, ignoreAttributes, collationExternalInfo.configInfo.c_str())) { + tt->texttype_flags = attributes; return true; } } diff --git a/src/jrd/SimilarToMatcher.h b/src/jrd/SimilarToMatcher.h deleted file mode 100644 index c931ac5c23..0000000000 --- a/src/jrd/SimilarToMatcher.h +++ /dev/null @@ -1,1919 +0,0 @@ -/* - * PROGRAM: JRD International support - * MODULE: SimilarToMatcher.h - * DESCRIPTION: SIMILAR TO predicate - * - * The contents of this file are subject to the Initial - * Developer's Public License Version 1.0 (the "License"); - * you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. - * - * Software distributed under the License is distributed AS IS, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. - * See the License for the specific language governing rights - * and limitations under the License. - * - * The Original Code was created by Adriano dos Santos Fernandes - * for the Firebird Open Source RDBMS project. - * - * Copyright (c) 2007 Adriano dos Santos Fernandes - * and all contributors signed below. - * - * All Rights Reserved. - * Contributor(s): ______________________________________. - */ - -#ifndef JRD_SIMILAR_TO_EVALUATOR_H -#define JRD_SIMILAR_TO_EVALUATOR_H - -#include "../jrd/intl_classes.h" -#include "../jrd/evl_string.h" - -// #define DEBUG_SIMILAR - -#ifdef DEBUG_SIMILAR -// #define RECURSIVE_SIMILAR // useless in production due to stack overflow -#endif - -namespace Firebird -{ - -template > -class SimilarToMatcher : public Jrd::PatternMatcher -{ -private: - typedef Jrd::CharSet CharSet; - typedef Jrd::TextType TextType; - - // This class is based on work of Zafir Anjum - // http://www.codeguru.com/Cpp/Cpp/string/regex/article.php/c2791 - // which has been derived from work by Henry Spencer. - // - // The original copyright notice follows: - // - // Copyright (c) 1986, 1993, 1995 by University of Toronto. - // Written by Henry Spencer. Not derived from licensed software. - // - // Permission is granted to anyone to use this software for any - // purpose on any computer system, and to redistribute it in any way, - // subject to the following restrictions: - // - // 1. The author is not responsible for the consequences of use of - // this software, no matter how awful, even if they arise - // from defects in it. - // - // 2. The origin of this software must not be misrepresented, either - // by explicit claim or by omission. - // - // 3. Altered versions must be plainly marked as such, and must not - // be misrepresented (by explicit claim or omission) as being - // the original software. - // - // 4. This notice must not be removed or altered. - class Evaluator : private StaticAllocator - { - public: - Evaluator(MemoryPool& pool, TextType* aTextType, - const UCHAR* patternStr, SLONG patternLen, - CharType aEscapeChar, bool aUseEscape); - - ~Evaluator() - { - delete[] branches; - } - - bool getResult(); - bool processNextChunk(const UCHAR* data, SLONG dataLen); - void reset(); - - private: - enum Op - { - opBranch, - opStart, - opEnd, - opRef, - opRepeatingRefStart, - opRepeatingRefEnd, - opNothing, - opAny, - opAnyOf, - opExactly, - opExactlyOne, // optimization for opExactly with a single character - // Implementation details of the non-recursive match - opRet, - opRepeatingRestore - // If new codes are added, shifts in MatchState codes may need to change. - }; - - struct Node - { - explicit Node(Op aOp, const CharType* aStr = NULL, SLONG aLen = 0) - : op(aOp), - str(aStr), - len(aLen), - str2(NULL), - len2(0), - str3(aStr), - len3(aLen), - str4(NULL), - len4(0), - ref(0), - branchNum(-1) - { - } - - Node(Op aOp, SLONG aLen1, SLONG aLen2, int aRef) - : op(aOp), - str(NULL), - len(aLen1), - str2(NULL), - len2(aLen2), - str3(NULL), - len3(0), - str4(NULL), - len4(0), - ref(aRef), - branchNum(-1) - { - } - - Node(Op aOp, int aRef) - : op(aOp), - str(NULL), - len(0), - str2(NULL), - len2(0), - str3(NULL), - len3(0), - str4(NULL), - len4(0), - ref(aRef), - branchNum(-1) - { - } - - Node(const Node& node) - : op(node.op), - str(node.str), - len(node.len), - str2(node.str2), - len2(node.len2), - str3(node.str3), - len3(node.len3), - str4(node.str4), - len4(node.len4), - ref(node.ref), - branchNum(node.branchNum) - { - } - -#ifdef DEBUG_SIMILAR - void dump(string& text, int i) const - { - string temp; - - switch (op) - { - case opBranch: - if (branchNum == -1) - temp.printf("opBranch(%d)", i + ref); - else - temp.printf("opBranch(%d, %d)", i + ref, branchNum); - break; - - case opStart: - temp = "opStart"; - break; - - case opEnd: - temp = "opEnd"; - break; - - case opRef: - if (branchNum == -1) - temp.printf("opRef(%d)", i + ref); - else - temp.printf("opRef(%d, %d)", i + ref, branchNum); - break; - - case opRepeatingRefStart: - temp.printf("opRepeatingRefStart(%d, %d)", i + ref, len); - break; - - case opRepeatingRefEnd: - temp.printf("opRepeatingRefEnd(%d)", i + ref); - break; - - case opNothing: - temp = "opNothing"; - break; - - case opAny: - temp = "opAny"; - break; - - case opAnyOf: - temp.printf("opAnyOf(%.*s, %d, %.*s, %d, %.*s, %d, %.*s, %d)", - len, str, len, len2, str2, len2, len3, str3, len3, len4, str4, len4); - break; - - case opExactly: - temp.printf("opExactly(%.*s, %d)", len, str, len); - break; - - case opExactlyOne: - temp.printf("opExactlyOne(%.*s)", len, str); - break; - - case opRet: - temp.printf("opRet"); - break; - - case opRepeatingRestore: - temp.printf("opRepeatingRestore"); - break; - - default: - temp = "unknown"; - break; - } - - text.printf("%d: %s", i, temp.c_str()); - } -#endif // DEBUG_SIMILAR - - Op op; - const CharType* str; - SLONG len; - const UCHAR* str2; - SLONG len2; - const CharType* str3; - SLONG len3; - const UCHAR* str4; - SLONG len4; - int ref; - int branchNum; - }; - -#ifndef RECURSIVE_SIMILAR - // Struct used to evaluate expressions without recursion. - // Represents local variables to implement a "virtual stack". - struct Scope - { - inline explicit Scope(const Node* ai) - : i(ai), - save(NULL) - { - } - - inline void operator =(const Node* ai) - { - i = ai; - save = NULL; - } - - const Node* i; - const CharType* save; - }; - - // Stack for recursion emulation. - template - class SimpleStack - { - public: - SimpleStack() - : size(INCREASE_FACTOR) - { - data = FB_NEW_POOL(*getDefaultMemoryPool()) UCHAR[(size + 1) * sizeof(T)]; - back = (T*) FB_ALIGN(data.get(), sizeof(T)); - end = back + size; - - // 'back' starts before initial element, then always points to the last pushed element. - --back; - } - - template - inline void push(T2 node) - { - // If the limit is reached, resize. - if (++back == end) - { - unsigned newSize = size + INCREASE_FACTOR; - UCHAR* newData = FB_NEW_POOL(*getDefaultMemoryPool()) UCHAR[(newSize + 1) * sizeof(T)]; - - T* p = (T*) FB_ALIGN(newData, sizeof(T)); - memcpy(p, end - size, size * sizeof(T)); - - back = p + size; - end = p + newSize; - size = newSize; - - data.reset(newData); - } - - *back = node; - } - - inline T pop() - { - fb_assert(getCount() > 0); - return *back--; - } - - inline T* begin() const - { - return (T*) FB_ALIGN(data.get(), sizeof(T)); - } - - inline FB_SIZE_T getCount() const - { - return (back + 1) - begin(); - } - - public: - T* back; - - private: - static const unsigned INCREASE_FACTOR = 50; - unsigned size; - AutoPtr data; - T* end; - }; -#endif // RECURSIVE_SIMILAR - - static const int FLAG_NOT_EMPTY = 1; // known never to match empty string - static const int FLAG_EXACTLY = 2; // non-escaped string - - private: - void parseExpr(int* flagp); - void parseTerm(int* flagp); - void parseFactor(int* flagp); - void parsePrimary(int* flagp); - bool isRep(CharType c) const; - - CharType canonicalChar(int ch) const - { - return *reinterpret_cast(textType->getCanonicalChar(ch)); - } - -#ifdef DEBUG_SIMILAR - void dump() const; -#endif - - private: -#ifdef RECURSIVE_SIMILAR - bool match(int start); -#else - bool match(); -#endif - - private: - static SLONG notInSet(const CharType* str, SLONG strLen, - const CharType* set, SLONG setLen); - - private: - struct Range - { - unsigned start; - unsigned length; - }; - -#ifdef DEBUG_SIMILAR - Array debugLog; - int debugLevel; -#endif - - TextType* textType; - CharType escapeChar; - bool useEscape; - HalfStaticArray buffer; - const UCHAR* originalPatternStr; - SLONG originalPatternLen; - StrConverter patternCvt; - CharSet* charSet; - Array nodes; - const CharType* patternStart; - const CharType* patternEnd; - const CharType* patternPos; - const CharType* bufferStart; - const CharType* bufferEnd; - const CharType* bufferPos; - CharType metaCharacters[15]; - - public: - unsigned branchNum; - Range* branches; - }; - -public: - SimilarToMatcher(MemoryPool& pool, TextType* ttype, const UCHAR* str, - SLONG strLen, CharType escape, bool useEscape) - : PatternMatcher(pool, ttype), - evaluator(pool, ttype, str, strLen, escape, useEscape) - { - } - - void reset() - { - evaluator.reset(); - } - - bool result() - { - return evaluator.getResult(); - } - - bool process(const UCHAR* str, SLONG length) - { - return evaluator.processNextChunk(str, length); - } - - unsigned getNumBranches() - { - return evaluator.branchNum; - } - - void getBranchInfo(unsigned n, unsigned* start, unsigned* length) - { - fb_assert(n <= evaluator.branchNum); - *start = evaluator.branches[n].start; - *length = evaluator.branches[n].length; - } - - static SimilarToMatcher* create(MemoryPool& pool, TextType* ttype, - const UCHAR* str, SLONG length, const UCHAR* escape, SLONG escapeLen) - { - StrConverter cvt_escape(pool, ttype, escape, escapeLen); - - return FB_NEW_POOL(pool) SimilarToMatcher(pool, ttype, str, length, - (escape ? *reinterpret_cast(escape) : 0), escapeLen != 0); - } - - static bool evaluate(MemoryPool& pool, TextType* ttype, const UCHAR* s, SLONG sl, - const UCHAR* p, SLONG pl, const UCHAR* escape, SLONG escapeLen) - { - StrConverter cvt_escape(pool, ttype, escape, escapeLen); - - Evaluator evaluator(pool, ttype, p, pl, - (escape ? *reinterpret_cast(escape) : 0), escapeLen != 0); - evaluator.processNextChunk(s, sl); - return evaluator.getResult(); - } - -private: - Evaluator evaluator; -}; - - -template -SimilarToMatcher::Evaluator::Evaluator( - MemoryPool& pool, TextType* aTextType, - const UCHAR* patternStr, SLONG patternLen, - CharType aEscapeChar, bool aUseEscape) - : StaticAllocator(pool), -#ifdef DEBUG_SIMILAR - debugLog(pool), - debugLevel(-1), -#endif - textType(aTextType), - escapeChar(aEscapeChar), - useEscape(aUseEscape), - buffer(pool), - originalPatternStr(patternStr), - originalPatternLen(patternLen), - patternCvt(pool, textType, patternStr, patternLen), - charSet(textType->getCharSet()), - nodes(pool), - branchNum(0) -{ - fb_assert(patternLen % sizeof(CharType) == 0); - patternLen /= sizeof(CharType); - - CharType* p = metaCharacters; - *p++ = canonicalChar(TextType::CHAR_CIRCUMFLEX); - *p++ = canonicalChar(TextType::CHAR_MINUS); - *p++ = canonicalChar(TextType::CHAR_UNDERLINE); - *p++ = canonicalChar(TextType::CHAR_PERCENT); - *p++ = canonicalChar(TextType::CHAR_OPEN_BRACKET); - *p++ = canonicalChar(TextType::CHAR_CLOSE_BRACKET); - *p++ = canonicalChar(TextType::CHAR_OPEN_PAREN); - *p++ = canonicalChar(TextType::CHAR_CLOSE_PAREN); - *p++ = canonicalChar(TextType::CHAR_OPEN_BRACE); - *p++ = canonicalChar(TextType::CHAR_CLOSE_BRACE); - *p++ = canonicalChar(TextType::CHAR_VERTICAL_BAR); - *p++ = canonicalChar(TextType::CHAR_QUESTION_MARK); - *p++ = canonicalChar(TextType::CHAR_PLUS); - *p++ = canonicalChar(TextType::CHAR_ASTERISK); - if (useEscape) - *p++ = escapeChar; - else - *p++ = canonicalChar(TextType::CHAR_ASTERISK); // just repeat something - fb_assert(p - metaCharacters == FB_NELEM(metaCharacters)); - - patternStart = patternPos = (const CharType*) patternStr; - patternEnd = patternStart + patternLen; - - nodes.push(Node(opStart)); - - int flags; - parseExpr(&flags); - - nodes.push(Node(opEnd)); - -#ifdef DEBUG_SIMILAR - dump(); -#endif - - // Check for proper termination. - if (patternPos < patternEnd) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - branches = FB_NEW_POOL(pool) Range[branchNum + 1]; - - reset(); -} - - -template -bool SimilarToMatcher::Evaluator::getResult() -{ - const UCHAR* str = buffer.begin(); - SLONG len = buffer.getCount(); - - // note that StrConverter changes str and len variables - StrConverter cvt(pool, textType, str, len); - fb_assert(len % sizeof(CharType) == 0); - - bufferStart = bufferPos = (const CharType*) str; - bufferEnd = bufferStart + len / sizeof(CharType); - -#ifdef DEBUG_SIMILAR - debugLog.clear(); - debugLevel = -1; -#endif - - const bool matched = -#ifdef RECURSIVE_SIMILAR - match(0); -#else - match(); -#endif - -#ifdef DEBUG_SIMILAR - if (matched) - { - for (unsigned i = 0; i <= branchNum; ++i) - { - string x; - x.printf("%d: %d, %d\n", i, branches[i].start, branches[i].length); - debugLog.add(x.c_str(), x.length()); - } - - debugLog.add('\0'); - - gds__log("\n%s", debugLog.begin()); - } -#endif // DEBUG_SIMILAR - - return matched; -} - - -template -bool SimilarToMatcher::Evaluator::processNextChunk(const UCHAR* data, SLONG dataLen) -{ - const FB_SIZE_T pos = buffer.getCount(); - memcpy(buffer.getBuffer(pos + dataLen) + pos, data, dataLen); - return true; -} - - -template -void SimilarToMatcher::Evaluator::reset() -{ - buffer.shrink(0); - - memset(branches, 0, sizeof(Range) * (branchNum + 1)); -} - - -template -void SimilarToMatcher::Evaluator::parseExpr(int* flagp) -{ - *flagp = FLAG_NOT_EMPTY; - - bool first = true; - Array refs; - int start; - - while (first || (patternPos < patternEnd && *patternPos == canonicalChar(TextType::CHAR_VERTICAL_BAR))) - { - if (first) - first = false; - else - ++patternPos; - - int thisBranchNum = branchNum; - start = nodes.getCount(); - nodes.push(Node(opBranch)); - nodes.back().branchNum = thisBranchNum; - - int flags; - parseTerm(&flags); - *flagp &= ~(~flags & FLAG_NOT_EMPTY); - *flagp |= flags; - - refs.push(nodes.getCount()); - nodes.push(Node(opRef)); - nodes.back().branchNum = thisBranchNum; - - nodes[start].ref = nodes.getCount() - start; - } - - nodes[start].ref = 0; - - for (Array::iterator i = refs.begin(); i != refs.end(); ++i) - nodes[*i].ref = nodes.getCount() - *i; -} - - -template -void SimilarToMatcher::Evaluator::parseTerm(int* flagp) -{ - *flagp = 0; - - bool first = true; - CharType c; - int flags; - - while ((patternPos < patternEnd) && - (c = *patternPos) != canonicalChar(TextType::CHAR_VERTICAL_BAR) && - c != canonicalChar(TextType::CHAR_CLOSE_PAREN)) - { - parseFactor(&flags); - - *flagp |= flags & FLAG_NOT_EMPTY; - - if (first) - { - *flagp |= flags; - first = false; - } - } - - if (first) - nodes.push(Node(opNothing)); -} - - -template -void SimilarToMatcher::Evaluator::parseFactor(int* flagp) -{ - int atomPos = nodes.getCount(); - - int flags; - parsePrimary(&flags); - - CharType op; - - if (patternPos >= patternEnd || !isRep((op = *patternPos))) - { - *flagp = flags; - return; - } - - if (!(flags & FLAG_NOT_EMPTY) && op != canonicalChar(TextType::CHAR_QUESTION_MARK)) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - // If the last primary is a string, split the last character - if (flags & FLAG_EXACTLY) - { - fb_assert(nodes.back().op == opExactly || nodes.back().op == opExactlyOne); - - if (nodes.back().op == opExactly && nodes.back().len > 1) - { - Node last = nodes.back(); - last.op = opExactlyOne; - last.str += nodes.back().len - 1; - last.len = 1; - - --nodes.back().len; - atomPos = nodes.getCount(); - nodes.push(last); - } - } - - fb_assert( - op == canonicalChar(TextType::CHAR_ASTERISK) || - op == canonicalChar(TextType::CHAR_PLUS) || - op == canonicalChar(TextType::CHAR_QUESTION_MARK) || - op == canonicalChar(TextType::CHAR_OPEN_BRACE)); - - if (op == canonicalChar(TextType::CHAR_ASTERISK)) - { - *flagp = 0; - nodes.insert(atomPos, Node(opBranch, nodes.getCount() - atomPos + 2)); - nodes.push(Node(opRef, atomPos - nodes.getCount())); - nodes.push(Node(opBranch)); - } - else if (op == canonicalChar(TextType::CHAR_PLUS)) - { - *flagp = FLAG_NOT_EMPTY; - nodes.push(Node(opBranch, 2)); - nodes.push(Node(opRef, atomPos - nodes.getCount())); - nodes.push(Node(opBranch)); - } - else if (op == canonicalChar(TextType::CHAR_QUESTION_MARK)) - { - *flagp = 0; - nodes.insert(atomPos, Node(opBranch, nodes.getCount() - atomPos + 1)); - nodes.push(Node(opBranch)); - } - else if (op == canonicalChar(TextType::CHAR_OPEN_BRACE)) - { - ++patternPos; - - UCharBuffer dummy; - const UCHAR* p = originalPatternStr + - charSet->substring(originalPatternLen, originalPatternStr, - originalPatternLen, dummy.getBuffer(originalPatternLen), - 1, patternPos - patternStart); - ULONG size = 0; - bool comma = false; - string s1, s2; - bool ok; - - while ((ok = IntlUtil::readOneChar(charSet, &p, originalPatternStr + originalPatternLen, &size))) - { - if (*patternPos == canonicalChar(TextType::CHAR_CLOSE_BRACE)) - { - if (s1.isEmpty()) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - break; - } - else if (*patternPos == canonicalChar(TextType::CHAR_COMMA)) - { - if (comma) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - comma = true; - } - else - { - ULONG ch = 0; - charSet->getConvToUnicode().convert(size, p, sizeof(ch), reinterpret_cast(&ch)); - - if (ch >= '0' && ch <= '9') - { - if (comma) - s2 += (char) ch; - else - s1 += (char) ch; - } - else - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - } - - ++patternPos; - } - - if (!ok || s1.length() > 9 || s2.length() > 9) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - const int n1 = atoi(s1.c_str()); - const int n2 = s2.isEmpty() ? (comma ? INT_MAX : n1) : atoi(s2.c_str()); - - if (n2 < n1) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - *flagp = n1 == 0 ? 0 : FLAG_NOT_EMPTY; - - if (n1 == 0 && n2 == INT_MAX) - { - // Tranforms x{0,} to x* - nodes.insert(atomPos, Node(opBranch, nodes.getCount() - atomPos + 2)); - nodes.push(Node(opRef, atomPos - nodes.getCount())); - nodes.push(Node(opBranch)); - } - else - { - if (n1 == 0) - { - // Tranforms x{,n} to (x?){n} - nodes.insert(atomPos, Node(opBranch, nodes.getCount() - atomPos + 1)); - nodes.push(Node(opBranch)); - } - - int exprPos = atomPos + 1; - int exprSize = nodes.getCount() - exprPos + 1; - - nodes.insert(atomPos, Node(opRepeatingRefStart, (n1 == 0 ? n2 : n1), 0, - nodes.getCount() - atomPos + 1)); - nodes.push(Node(opRepeatingRefEnd, atomPos - nodes.getCount())); - - if (n2 != n1 && n1 != 0) - { - if (n2 == INT_MAX) - { - // Tranforms x{n,} to x{n}x* - - nodes.push(Node(opBranch, exprSize + 2)); - - for (int i = 0; i < exprSize; ++i) - { - Node copy(nodes[exprPos + i]); - nodes.push(copy); - } - - nodes.push(Node(opRef, -exprSize - 1)); - nodes.push(Node(opBranch)); - } - else - { - // Tranforms x{n,m} to x{n}(x?){m-n} - - nodes.push(Node(opRepeatingRefStart, n2 - n1, 0, exprSize + 3)); - nodes.push(Node(opBranch, exprSize + 1)); - - for (int i = 0; i < exprSize; ++i) - { - Node copy(nodes[exprPos + i]); - nodes.push(copy); - } - - nodes.push(Node(opBranch)); - nodes.push(Node(opRepeatingRefEnd, -exprSize -3)); - } - } - } - } - - ++patternPos; - - if (patternPos < patternEnd && isRep(*patternPos)) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); -} - - -template -void SimilarToMatcher::Evaluator::parsePrimary(int* flagp) -{ - *flagp = 0; - - const CharType op = *patternPos++; - - if (op == canonicalChar(TextType::CHAR_UNDERLINE)) - { - nodes.push(Node(opAny)); - *flagp |= FLAG_NOT_EMPTY; - } - else if (op == canonicalChar(TextType::CHAR_PERCENT)) - { - nodes.push(Node(opBranch, 3)); - nodes.push(Node(opAny)); - nodes.push(Node(opRef, -2)); - nodes.push(Node(opBranch)); - - *flagp = 0; - return; - } - else if (op == canonicalChar(TextType::CHAR_OPEN_BRACKET)) - { - nodes.push(Node(opAnyOf)); - - HalfStaticArray charsBuffer; - HalfStaticArray rangeBuffer; - - Node& node = nodes.back(); - const CharType** nodeChars = &node.str; - SLONG* nodeCharsLen = &node.len; - const UCHAR** nodeRange = &node.str2; - SLONG* nodeRangeLen = &node.len2; - - bool but = false; - - do - { - if (patternPos >= patternEnd) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - bool range = false; - bool charClass = false; - - if (useEscape && *patternPos == escapeChar) - { - if (++patternPos >= patternEnd) - status_exception::raise(Arg::Gds(isc_escape_invalid)); - - if (*patternPos != escapeChar && - notInSet(patternPos, 1, metaCharacters, FB_NELEM(metaCharacters)) != 0) - { - status_exception::raise(Arg::Gds(isc_escape_invalid)); - } - - if (patternPos + 1 < patternEnd) - range = (patternPos[1] == canonicalChar(TextType::CHAR_MINUS)); - } - else - { - if (*patternPos == canonicalChar(TextType::CHAR_OPEN_BRACKET)) - charClass = true; - else if (*patternPos == canonicalChar(TextType::CHAR_CIRCUMFLEX)) - { - if (but) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - but = true; - - CharType* p = (CharType*) alloc(charsBuffer.getCount() * sizeof(CharType)); - memcpy(p, charsBuffer.begin(), charsBuffer.getCount() * sizeof(CharType)); - *nodeChars = p; - - *nodeCharsLen = charsBuffer.getCount(); - - if (rangeBuffer.getCount() > 0) - { - UCHAR* p = (UCHAR*) alloc(rangeBuffer.getCount()); - memcpy(p, rangeBuffer.begin(), rangeBuffer.getCount()); - *nodeRange = p; - } - - *nodeRangeLen = rangeBuffer.getCount(); - - charsBuffer.clear(); - rangeBuffer.clear(); - - nodeChars = &node.str3; - nodeCharsLen = &node.len3; - nodeRange = &node.str4; - nodeRangeLen = &node.len4; - - ++patternPos; - continue; - } - else if (patternPos + 1 < patternEnd) - range = (patternPos[1] == canonicalChar(TextType::CHAR_MINUS)); - } - - if (charClass) - { - if (++patternPos >= patternEnd || *patternPos != canonicalChar(TextType::CHAR_COLON)) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - const CharType* start = ++patternPos; - - while (patternPos < patternEnd && *patternPos != canonicalChar(TextType::CHAR_COLON)) - ++patternPos; - - const SLONG len = patternPos++ - start; - - if (patternPos >= patternEnd || *patternPos++ != canonicalChar(TextType::CHAR_CLOSE_BRACKET)) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - typedef const UCHAR* (TextType::*GetCanonicalFunc)(int*) const; - - static const GetCanonicalFunc alNum[] = {&TextType::getCanonicalUpperLetters, - &TextType::getCanonicalLowerLetters, &TextType::getCanonicalNumbers, NULL}; - static const GetCanonicalFunc alpha[] = {&TextType::getCanonicalUpperLetters, - &TextType::getCanonicalLowerLetters, NULL}; - static const GetCanonicalFunc digit[] = {&TextType::getCanonicalNumbers, NULL}; - static const GetCanonicalFunc lower[] = {&TextType::getCanonicalLowerLetters, NULL}; - static const GetCanonicalFunc space[] = {&TextType::getCanonicalSpace, NULL}; - static const GetCanonicalFunc upper[] = {&TextType::getCanonicalUpperLetters, NULL}; - static const GetCanonicalFunc whitespace[] = {&TextType::getCanonicalWhiteSpaces, NULL}; - - struct - { - const GetCanonicalFunc* funcs; - const ULONG nameLen; // in bytes, not characters because all functions accept length in bytes - const USHORT name[10]; - } static const classes[] = - { // Names are in utf16 in order not to convert them every time for comparison and thus save some CPU - {alNum, 10, {'A','L','N','U','M'}}, - {alpha, 10, {'A','L','P','H','A'}}, - {digit, 10, {'D','I','G','I','T'}}, - {lower, 10, {'L','O','W','E','R'}}, - {space, 10, {'S','P','A','C','E'}}, - {upper, 10, {'U','P','P','E','R'}}, - {whitespace, 20, {'W','H','I','T','E','S','P','A','C','E'}} - }; - - // Get the exact original substring correspondent to the canonical bytes. - HalfStaticArray classNameStr( - len * charSet->maxBytesPerChar()); - ULONG classNameStrLen = charSet->substring(originalPatternLen, originalPatternStr, - classNameStr.getCapacity(), classNameStr.begin(), start - patternStart, len); - - // And then convert it to UTF-16. - HalfStaticArray classNameUtf16( - len * sizeof(ULONG)); - ULONG classNameUtf16Len = charSet->getConvToUnicode().convert( - classNameStrLen, classNameStr.begin(), - classNameUtf16.getCapacity() * sizeof(USHORT), classNameUtf16.begin()); - - // Bring class name to uppercase for case-insensitivity. - // Do it in UTF-16 because original collation can have no uppercase conversion. - classNameUtf16Len = Jrd::UnicodeUtil::utf16UpperCase( - classNameUtf16Len, classNameUtf16.begin(), - classNameUtf16.getCapacity() * sizeof(USHORT), classNameUtf16.begin(), NULL); - int classN; - - for (classN = 0; classN < FB_NELEM(classes); ++classN) - { - INTL_BOOL errorFlag; - - if (Jrd::UnicodeUtil::utf16Compare(classNameUtf16Len, classNameUtf16.begin(), - classes[classN].nameLen, classes[classN].name, &errorFlag) == 0) - { - for (const GetCanonicalFunc* func = classes[classN].funcs; *func; ++func) - { - int count; - const CharType* canonic = (const CharType*) (textType->**func)(&count); - charsBuffer.push(canonic, count); - } - - break; - } - } - - if (classN >= FB_NELEM(classes)) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - } - else - { - charsBuffer.push(*patternPos++); - - if (range) - { - --patternPos; // go back to first char - - UCHAR c[sizeof(ULONG)]; - ULONG len = charSet->substring(originalPatternLen, originalPatternStr, - sizeof(c), c, patternPos - patternStart, 1); - - rangeBuffer.push(len); - FB_SIZE_T rangeCount = rangeBuffer.getCount(); - memcpy(rangeBuffer.getBuffer(rangeCount + len) + rangeCount, &c, len); - - ++patternPos; // character - ++patternPos; // minus - - if (patternPos >= patternEnd) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - if (useEscape && *patternPos == escapeChar) - { - if (++patternPos >= patternEnd) - status_exception::raise(Arg::Gds(isc_escape_invalid)); - - if (*patternPos != escapeChar && - notInSet(patternPos, 1, metaCharacters, FB_NELEM(metaCharacters)) != 0) - { - status_exception::raise(Arg::Gds(isc_escape_invalid)); - } - } - - len = charSet->substring(originalPatternLen, originalPatternStr, - sizeof(c), c, patternPos - patternStart, 1); - - rangeBuffer.push(len); - rangeCount = rangeBuffer.getCount(); - memcpy(rangeBuffer.getBuffer(rangeCount + len) + rangeCount, &c, len); - - charsBuffer.push(*patternPos++); - } - } - - if (patternPos >= patternEnd) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - } while (*patternPos != canonicalChar(TextType::CHAR_CLOSE_BRACKET)); - - CharType* p = (CharType*) alloc(charsBuffer.getCount() * sizeof(CharType)); - memcpy(p, charsBuffer.begin(), charsBuffer.getCount() * sizeof(CharType)); - *nodeChars = p; - - *nodeCharsLen = charsBuffer.getCount(); - - if (rangeBuffer.getCount() > 0) - { - UCHAR* r = (UCHAR*) alloc(rangeBuffer.getCount()); - memcpy(r, rangeBuffer.begin(), rangeBuffer.getCount()); - *nodeRange = r; - } - - *nodeRangeLen = rangeBuffer.getCount(); - - ++patternPos; - *flagp |= FLAG_NOT_EMPTY; - } - else if (op == canonicalChar(TextType::CHAR_OPEN_PAREN)) - { - int flags; - parseExpr(&flags); - - ++branchNum; // This is used for the trace stuff. - - if (patternPos >= patternEnd || *patternPos++ != canonicalChar(TextType::CHAR_CLOSE_PAREN)) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - *flagp |= flags & FLAG_NOT_EMPTY; - } - else if (useEscape && op == escapeChar) - { - if (patternPos >= patternEnd) - status_exception::raise(Arg::Gds(isc_escape_invalid)); - - if (*patternPos != escapeChar && - notInSet(patternPos, 1, metaCharacters, FB_NELEM(metaCharacters)) != 0) - { - status_exception::raise(Arg::Gds(isc_escape_invalid)); - } - - nodes.push(Node(opExactlyOne, patternPos++, 1)); - *flagp |= FLAG_NOT_EMPTY; - } - else - { - --patternPos; - - const SLONG len = notInSet(patternPos, patternEnd - patternPos, - metaCharacters, FB_NELEM(metaCharacters)); - - if (len == 0) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - *flagp |= FLAG_NOT_EMPTY | FLAG_EXACTLY; - - nodes.push(Node((len == 1 ? opExactlyOne : opExactly), patternPos, len)); - patternPos += len; - } -} - - -template -bool SimilarToMatcher::Evaluator::isRep(CharType c) const -{ - return (c == canonicalChar(TextType::CHAR_ASTERISK) || - c == canonicalChar(TextType::CHAR_PLUS) || - c == canonicalChar(TextType::CHAR_QUESTION_MARK) || - c == canonicalChar(TextType::CHAR_OPEN_BRACE)); -} - - -#ifdef DEBUG_SIMILAR -template -void SimilarToMatcher::Evaluator::dump() const -{ - string text; - - for (unsigned i = 0; i < nodes.getCount(); ++i) - { - string type; - nodes[i].dump(type, i); - - string s; - s.printf("%s%s", (i > 0 ? ", " : ""), type.c_str()); - - text += s; - } - - gds__log("%s", text.c_str()); -} -#endif // DEBUG_SIMILAR - - -template -#ifdef RECURSIVE_SIMILAR -bool SimilarToMatcher::Evaluator::match(int start) -{ -#ifdef DEBUG_SIMILAR - AutoSetRestore autoDebugLevel(&debugLevel, debugLevel + 1); -#endif - - for (int i = start;; ++i) - { - const Node* node = &nodes[i]; - -#ifdef DEBUG_SIMILAR - string s; - node->dump(s, i); - - for (int debugLevelI = 0; debugLevelI < debugLevel; ++debugLevelI) - s = " " + s; - - s = "\n" + s; - debugLog.add(s.c_str(), s.length()); -#endif - - switch (node->op) - { - case opBranch: - { - const CharType* const save = bufferPos; - - while (true) - { - if (node->branchNum != -1) - branches[node->branchNum].start = save - bufferStart; - - if (match(i + 1)) - return true; - - bufferPos = save; - - if (node->ref == 0) - return false; - - i += node->ref; - node = &nodes[i]; - - if (node->ref == 0) - break; - -#ifdef DEBUG_SIMILAR - node->dump(s, i); - - for (int debugLevelI = 0; debugLevelI < debugLevel; ++debugLevelI) - s = " " + s; - - s = "\n" + s; - debugLog.add(s.c_str(), s.length()); -#endif - } - - break; - } - - case opStart: - if (bufferPos != bufferStart) - return false; - break; - - case opEnd: - return (bufferPos == bufferEnd); - - case opRef: - if (node->branchNum != -1) - { - fb_assert(unsigned(node->branchNum) <= branchNum); - branches[node->branchNum].length = - bufferPos - bufferStart - branches[node->branchNum].start; - } - - if (node->ref == 1) // avoid recursion - break; - return match(i + node->ref); - - //// FIXME: opRepeatingRefStart, opRepeatingRefEnd - - case opNothing: - break; - - case opAny: -#ifdef DEBUG_SIMILAR - if (bufferPos >= bufferEnd) - s = " -> "; - else - s.printf(" -> %d", *bufferPos); - debugLog.add(s.c_str(), s.length()); -#endif - - if (bufferPos >= bufferEnd) - return false; - ++bufferPos; - break; - - case opAnyOf: -#ifdef DEBUG_SIMILAR - if (bufferPos >= bufferEnd) - s = " -> "; - else - s.printf(" -> %d", *bufferPos); - debugLog.add(s.c_str(), s.length()); -#endif - - if (bufferPos >= bufferEnd) - return false; - - if (notInSet(bufferPos, 1, node->str, node->len) != 0) - { - const UCHAR* const end = node->str2 + node->len2; - const UCHAR* p = node->str2; - - while (p < end) - { - UCHAR c[sizeof(ULONG)]; - ULONG len = charSet->substring(buffer.getCount(), buffer.begin(), - sizeof(c), c, bufferPos - bufferStart, 1); - - if (textType->compare(len, c, p[0], p + 1) >= 0 && - textType->compare(len, c, p[1 + p[0]], p + 2 + p[0]) <= 0) - { - break; - } - - p += 2 + p[0] + p[1 + p[0]]; - } - - if (node->len + node->len2 != 0 && p >= end) - return false; - } - - if (notInSet(bufferPos, 1, node->str3, node->len3) == 0) - return false; - else - { - const UCHAR* const end = node->str4 + node->len4; - const UCHAR* p = node->str4; - - while (p < end) - { - UCHAR c[sizeof(ULONG)]; - const ULONG len = charSet->substring(buffer.getCount(), buffer.begin(), - sizeof(c), c, bufferPos - bufferStart, 1); - - if (textType->compare(len, c, p[0], p + 1) >= 0 && - textType->compare(len, c, p[1 + p[0]], p + 2 + p[0]) <= 0) - { - break; - } - - p += 2 + p[0] + p[1 + p[0]]; - } - - if (p < end) - return false; - } - - ++bufferPos; - break; - - case opExactly: - if (bufferEnd - bufferPos >= node->len && - memcmp(node->str, bufferPos, node->len * sizeof(CharType)) == 0) - { - bufferPos += node->len; - break; - } - else - return false; - - case opExactlyOne: - if (bufferEnd - bufferPos >= 1 && *node->str == *bufferPos) - { - bufferPos += node->len; - break; - } - else - return false; - - default: - fb_assert(false); - return false; - } - } - - return true; -} -#else -bool SimilarToMatcher::Evaluator::match() -{ - // Left shift by 4 to OR MatchState's and Op's without additional runtime shifts. - static const unsigned MATCH_STATE_SHIFT = 4; - - enum MatchState - { - msIterating = 0x00 << MATCH_STATE_SHIFT, - msReturningFalse = 0x01 << MATCH_STATE_SHIFT, - msReturningTrue = 0x02 << MATCH_STATE_SHIFT, - msReturningMask = (msReturningFalse | msReturningTrue) - }; - - SimpleStack scopeStack; - - // Add special node to return without needing additional comparison after popping - // the stack on each return. - Node nodeRet(opRet); - scopeStack.push(&nodeRet); - - scopeStack.push(nodes.begin()); - - MatchState state = msIterating; - - SimpleStack repeatStack; - SLONG repeatCount = 0; - Node nodeRepeatingRestore(opRepeatingRestore); - - while (true) - { - fb_assert(scopeStack.getCount() > 0); - - Scope* const scope = scopeStack.back; - const Node* const node = scope->i; - -#ifdef DEBUG_SIMILAR - string debugText; - node->dump(debugText, (node == &nodeRet ? -1 : node - nodes.begin())); - - for (const CharType* p = bufferPos; p != bufferEnd; ++p) - { - string s; - s.printf(" %04d", *p); - debugText += s; - } - - debugText += "\nrepeat:"; - - for (const int* p = repeatStack.begin(); p <= repeatStack.back; ++p) - { - string s; - s.printf(" %d", *p); - debugText += s; - } - - { - string s; - s.printf(" %d", repeatCount); - debugText += s; - } - - debugText += "\nscope:"; - - for (const Scope* p = scopeStack.begin(); p <= scopeStack.back; ++p) - { - string s; - s.printf(" %d", - (p->i == &nodeRet ? - -1 : - (p->i == &nodeRepeatingRestore ? - -2 : - p->i - nodes.begin()))); - debugText += s; - } - - gds__log("%d, %s", state, debugText.c_str()); -#endif - -#define ENCODE_OP_STATE(op, state) ((op) | (state)) - - // Go directly to op and state with a single switch. - - switch (ENCODE_OP_STATE(node->op, state)) - { - case ENCODE_OP_STATE(opBranch, msIterating): - if (node->branchNum != -1) - branches[node->branchNum].start = bufferPos - bufferStart; - - scope->save = bufferPos; - - scopeStack.push(scope->i + 1); - continue; - - case ENCODE_OP_STATE(opBranch, msReturningFalse): - bufferPos = scope->save; - - if (node->ref != 0) - { - state = msIterating; - - scope->i += node->ref; - - if (scope->i->ref != 0) - { - scope->save = bufferPos; - - scopeStack.push(scope->i + 1); - continue; - } - } - - break; - - case ENCODE_OP_STATE(opBranch, msReturningTrue): - break; - - case ENCODE_OP_STATE(opStart, msIterating): - if (bufferPos != bufferStart) - state = msReturningFalse; - break; - - case ENCODE_OP_STATE(opEnd, msIterating): - state = (bufferPos == bufferEnd ? msReturningTrue : msReturningFalse); - break; - - case ENCODE_OP_STATE(opRef, msIterating): - if (node->branchNum != -1) - { - fb_assert(unsigned(node->branchNum) <= branchNum); - branches[node->branchNum].length = - bufferPos - bufferStart - branches[node->branchNum].start; - } - - scope->i += node->ref; - scope->save = NULL; - continue; - - case ENCODE_OP_STATE(opRef, msReturningFalse): - case ENCODE_OP_STATE(opRef, msReturningTrue): - break; - - case ENCODE_OP_STATE(opRepeatingRefStart, msIterating): - repeatStack.push(repeatCount); - repeatCount = node->len; - scopeStack.push(scope->i + node->ref); - continue; - - case ENCODE_OP_STATE(opRepeatingRefStart, msReturningFalse): - case ENCODE_OP_STATE(opRepeatingRefStart, msReturningTrue): - repeatCount = repeatStack.pop(); - break; - - case ENCODE_OP_STATE(opRepeatingRefEnd, msIterating): - if (repeatCount > 0) - { - --repeatCount; - scopeStack.push(scope->i + node->ref + 1); - } - else - { - repeatCount = repeatStack.pop(); - scopeStack.push(&nodeRepeatingRestore); - scopeStack.push(scope->i + 1); - } - - continue; - - case ENCODE_OP_STATE(opRepeatingRefEnd, msReturningFalse): - ++repeatCount; - break; - - case ENCODE_OP_STATE(opRepeatingRefEnd, msReturningTrue): - break; - - case ENCODE_OP_STATE(opRepeatingRestore, msReturningFalse): - case ENCODE_OP_STATE(opRepeatingRestore, msReturningTrue): - repeatStack.push(repeatCount); - repeatCount = -1; - break; - - case ENCODE_OP_STATE(opNothing, msIterating): - case ENCODE_OP_STATE(opNothing, msReturningFalse): - case ENCODE_OP_STATE(opNothing, msReturningTrue): - break; - - case ENCODE_OP_STATE(opAny, msIterating): - if (bufferPos >= bufferEnd) - state = msReturningFalse; - else - ++bufferPos; - break; - - case ENCODE_OP_STATE(opAnyOf, msIterating): - if (bufferPos >= bufferEnd) - state = msReturningFalse; - else - { - if (notInSet(bufferPos, 1, node->str, node->len) != 0) - { - const UCHAR* const end = node->str2 + node->len2; - const UCHAR* p = node->str2; - - while (p < end) - { - UCHAR c[sizeof(ULONG)]; - const ULONG len = charSet->substring(buffer.getCount(), buffer.begin(), - sizeof(c), c, bufferPos - bufferStart, 1); - - if (textType->compare(len, c, p[0], p + 1) >= 0 && - textType->compare(len, c, p[1 + p[0]], p + 2 + p[0]) <= 0) - { - break; - } - - p += 2 + p[0] + p[1 + p[0]]; - } - - if (node->len + node->len2 != 0 && p >= end) - { - state = msReturningFalse; - break; - } - } - - if (notInSet(bufferPos, 1, node->str3, node->len3) == 0) - state = msReturningFalse; - else - { - const UCHAR* const end = node->str4 + node->len4; - const UCHAR* p = node->str4; - - while (p < end) - { - UCHAR c[sizeof(ULONG)]; - const ULONG len = charSet->substring( - buffer.getCount(), buffer.begin(), - sizeof(c), c, bufferPos - bufferStart, 1); - - if (textType->compare(len, c, p[0], p + 1) >= 0 && - textType->compare(len, c, p[1 + p[0]], p + 2 + p[0]) <= 0) - { - break; - } - - p += 2 + p[0] + p[1 + p[0]]; - } - - if (p < end) - state = msReturningFalse; - } - } - - if (state == msIterating) - ++bufferPos; - break; - - case ENCODE_OP_STATE(opExactly, msIterating): - if (bufferEnd - bufferPos >= node->len && - memcmp(node->str, bufferPos, node->len * sizeof(CharType)) == 0) - { - bufferPos += node->len; - } - else - state = msReturningFalse; - break; - - case ENCODE_OP_STATE(opExactlyOne, msIterating): - if (bufferEnd - bufferPos >= 1 && *node->str == *bufferPos) - ++bufferPos; - else - state = msReturningFalse; - break; - - case ENCODE_OP_STATE(opRet, msReturningFalse): - case ENCODE_OP_STATE(opRet, msReturningTrue): - fb_assert(repeatStack.getCount() == 0); - return state == msReturningTrue; - - default: - fb_assert(false); - return false; - } - -#undef ENCODE_OP_STATE - - switch (state) - { - case msIterating: - ++scope->i; - break; - - case msReturningFalse: - case msReturningTrue: - scopeStack.pop(); - break; - - default: - break; - } - } - - fb_assert(false); - return false; -} -#endif - - -// Returns the number of characters up to first one present in set. -template -SLONG SimilarToMatcher::Evaluator::notInSet( - const CharType* str, SLONG strLen, const CharType* set, SLONG setLen) -{ - for (const CharType* begin = str; str - begin < strLen; ++str) - { - for (const CharType* p = set; p - set < setLen; ++p) - { - if (*p == *str) - return str - begin; - } - } - - return strLen; -} - - -// Given a regular expression R1#R2#R3 and the string S: -// - Find the shortest substring of S that matches R1 while the remainder (S23) matches R2R3; -// - Find the longest (S2) substring of S23 that matches R2 while the remainder matches R3; -// - Return S2. -template > -class SubstringSimilarMatcher : public Jrd::BaseSubstringSimilarMatcher -{ -private: - typedef Jrd::CharSet CharSet; - typedef Jrd::TextType TextType; - -public: - SubstringSimilarMatcher(MemoryPool& pool, TextType* ttype, - const UCHAR* patternStr, SLONG patternLen, CharType aEscapeChar) - : BaseSubstringSimilarMatcher(pool, ttype), - escapeChar(aEscapeChar), - originalPatternStr(patternStr), - originalPatternLen(patternLen), - patternCvt(pool, textType, patternStr, patternLen), - buffer(pool) - { - CharSet* charSet = textType->getCharSet(); - - // Make a new string without the . While doing it, get the byte - // length of each segment. - - UCharBuffer newExpr(originalPatternLen); - UCHAR* newExprPos = newExpr.begin(); - - const UCHAR* originalPatternEnd = originalPatternStr + originalPatternLen; - const UCHAR* originalPatternPos = originalPatternStr; - - const CharType* lastStart = reinterpret_cast(patternStr); - const CharType* end = lastStart + patternLen; - unsigned lengths[3]; - unsigned lengthsNum = 0; - UCHAR dummy[sizeof(ULONG) * 2]; - - for (const CharType* p = lastStart; p < end; ++p) - { - if (*p != escapeChar) - continue; - - if (++p >= end) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - if (*p == canonicalChar(TextType::CHAR_DOUBLE_QUOTE)) - { - if (lengthsNum >= 2) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - // Get the byte length since the last segment. - ULONG len = charSet->substring(originalPatternEnd - originalPatternPos, - originalPatternPos, newExpr.begin() + originalPatternLen - newExprPos, - newExprPos, 0, p - lastStart - 1); - - lengths[lengthsNum++] = len; - newExprPos += len; - originalPatternPos += len; - - // Advance two () characters. - originalPatternPos += charSet->substring(originalPatternEnd - originalPatternPos, - originalPatternPos, sizeof(dummy), dummy, 0, 2); - - lastStart = p + 1; // Register the start of the next segment. - } - } - - if (lengthsNum != 2) - status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - - // Get the byte length of the last segment. - lengths[2] = charSet->substring(originalPatternEnd - originalPatternPos, - originalPatternPos, newExpr.begin() + originalPatternLen - newExprPos, - newExprPos, 0, end - lastStart); - - // Construct the needed regular expressions. - - r1 = FB_NEW_POOL(pool) SimilarToMatcher(pool, ttype, - newExpr.begin(), lengths[0], escapeChar, true); - - r2 = FB_NEW_POOL(pool) SimilarToMatcher(pool, ttype, - newExpr.begin() + lengths[0], lengths[1], escapeChar, true); - - r3 = FB_NEW_POOL(pool) SimilarToMatcher(pool, ttype, - newExpr.begin() + lengths[0] + lengths[1], lengths[2], escapeChar, true); - - r23 = FB_NEW_POOL(pool) SimilarToMatcher(pool, ttype, - newExpr.begin() + lengths[0], lengths[1] + lengths[2], escapeChar, true); - } - - static SubstringSimilarMatcher* create(MemoryPool& pool, TextType* ttype, - const UCHAR* str, SLONG length, const UCHAR* escape, SLONG escapeLen) - { - StrConverter cvt_escape(pool, ttype, escape, escapeLen); - - return FB_NEW_POOL(pool) SubstringSimilarMatcher(pool, ttype, str, length, - *reinterpret_cast(escape)); - } - - void reset() - { - buffer.shrink(0); - - r1->reset(); - r2->reset(); - r3->reset(); - r23->reset(); - } - - bool result() - { - CharSet* charSet = textType->getCharSet(); - const UCHAR* p = buffer.begin(); - UCharBuffer temp(buffer.getCount()); - UCHAR dummy[sizeof(ULONG)]; - - // Find the shortest substring that matches R1 while the full expression matches R1R2R3. - do - { - r1->reset(); - r1->process(buffer.begin(), p - buffer.begin()); - - if (r1->result()) - { - // We have a initial substring matching R1. Let's see if the remainder matches R2R3. - - r23->reset(); - r23->process(p, buffer.end() - p); - - if (r23->result()) - { - // Now we start to find the longest substring that matches R2 while the - // remainder matches R3. Once we found it, it's the result string. - - // We already know its start, based on the substring that matched R1. - matchedStart = p - buffer.begin(); - - const UCHAR* p3 = buffer.end(); - SLONG charLen23 = -1; - memcpy(temp.begin(), p, p3 - p); - - while (true) - { - r2->reset(); - r2->process(temp.begin(), p3 - p); - - if (r2->result()) - { - r3->reset(); - r3->process(p3, buffer.end() - p3); - - if (r3->result()) - { - matchedLength = p3 - buffer.begin() - matchedStart; - return true; - } - } - - if (charLen23 == -1) - charLen23 = charSet->length(p3 - p, p, true); - - if (charLen23-- == 0) - break; - - // Shrink in one character the string to match R2. - // Move back one character to match R3. - p3 = p + charSet->substring(buffer.end() - p, p, temp.getCapacity(), - temp.begin(), 0, charLen23); - } - } - } - - // Advance a character. - p += charSet->substring(buffer.end() - p, p, sizeof(dummy), dummy, 0, 1); - } while (p < buffer.end()); - - return false; - } - - bool process(const UCHAR* str, SLONG length) - { - const FB_SIZE_T pos = buffer.getCount(); - memcpy(buffer.getBuffer(pos + length) + pos, str, length); - return true; - } - - // We return byte-base start and length. - void getResultInfo(unsigned* start, unsigned* length) - { - *start = matchedStart; - *length = matchedLength; - } - -private: - CharType canonicalChar(int ch) const - { - return *reinterpret_cast(textType->getCanonicalChar(ch)); - } - -private: - CharType escapeChar; - const UCHAR* originalPatternStr; - SLONG originalPatternLen; - StrConverter patternCvt; - HalfStaticArray buffer; - AutoPtr r1, r2, r3, r23; - unsigned matchedStart; - unsigned matchedLength; -}; - - -} // namespace Firebird - -#endif // JRD_SIMILAR_TO_EVALUATOR_H diff --git a/src/jrd/intl_classes.h b/src/jrd/intl_classes.h index c3be9620f8..e7279ab0d4 100644 --- a/src/jrd/intl_classes.h +++ b/src/jrd/intl_classes.h @@ -87,23 +87,12 @@ public: UpcaseConverter(MemoryPool& pool, TextType* obj, const UCHAR*& str, SLONG& len) : PrevConverter(pool, obj, str, len) { - if (len > (int) sizeof(tempBuffer)) - out_str = FB_NEW_POOL(pool) UCHAR[len]; - else - out_str = tempBuffer; - obj->str_to_upper(len, str, len, out_str); - str = out_str; - } - - ~UpcaseConverter() - { - if (out_str != tempBuffer) - delete[] out_str; + obj->str_to_upper(len, str, len, tempBuffer.getBuffer(len, false)); + str = tempBuffer.begin(); } private: - UCHAR tempBuffer[100]; - UCHAR* out_str; + Firebird::UCharBuffer tempBuffer; }; template @@ -115,29 +104,17 @@ public: { const SLONG out_len = len / obj->getCharSet()->minBytesPerChar() * obj->getCanonicalWidth(); - if (out_len > (int) sizeof(tempBuffer)) - out_str = FB_NEW_POOL(pool) UCHAR[out_len]; - else - out_str = tempBuffer; - if (str) { - len = obj->canonical(len, str, out_len, out_str) * obj->getCanonicalWidth(); - str = out_str; + len = obj->canonical(len, str, out_len, tempBuffer.getBuffer(out_len, false)) * obj->getCanonicalWidth(); + str = tempBuffer.begin(); } else len = 0; } - ~CanonicalConverter() - { - if (out_str != tempBuffer) - delete[] out_str; - } - private: - UCHAR tempBuffer[100]; - UCHAR* out_str; + Firebird::UCharBuffer tempBuffer; }; } // namespace Jrd diff --git a/src/jrd/replication/Manager.cpp b/src/jrd/replication/Manager.cpp index 2ae0742fd6..a8960c8328 100644 --- a/src/jrd/replication/Manager.cpp +++ b/src/jrd/replication/Manager.cpp @@ -50,50 +50,23 @@ TableMatcher::TableMatcher(MemoryPool& pool, const string& excludeFilter) : m_tables(pool) { - m_cs = FB_NEW_POOL(pool) charset; - m_tt = FB_NEW_POOL(pool) texttype; - - IntlUtil::initUtf8Charset(m_cs); - - string collAttributes("ICU-VERSION="); - collAttributes += Jrd::UnicodeUtil::getDefaultIcuVersion(); - IntlUtil::setupIcuAttributes(m_cs, collAttributes, "", collAttributes); - - UCharBuffer collAttributesBuffer; - collAttributesBuffer.push(reinterpret_cast(collAttributes.c_str()), - collAttributes.length()); - - if (!IntlUtil::initUnicodeCollation(m_tt, m_cs, "UNICODE", 0, collAttributesBuffer, "")) - raiseError("Cannot initialize UNICODE collation"); - - m_charSet = CharSet::createInstance(pool, 0, m_cs); - m_textType = FB_NEW_POOL(pool) TextType(0, m_tt, m_charSet); - if (includeFilter.hasData()) { - m_includeMatcher.reset(FB_NEW_POOL(pool) SimilarMatcher( - pool, m_textType, - (const UCHAR*) includeFilter.c_str(), - includeFilter.length(), - '\\', true)); + m_includeMatcher.reset(FB_NEW_POOL(pool) SimilarToRegex( + pool, true, + includeFilter.c_str(), includeFilter.length(), + "\\", 1)); } if (excludeFilter.hasData()) { - m_excludeMatcher.reset(FB_NEW_POOL(pool) SimilarMatcher( - pool, m_textType, - (const UCHAR*) excludeFilter.c_str(), - excludeFilter.length(), - '\\', true)); + m_excludeMatcher.reset(FB_NEW_POOL(pool) SimilarToRegex( + pool, true, + excludeFilter.c_str(), excludeFilter.length(), + "\\", 1)); } } -TableMatcher::~TableMatcher() -{ - if (m_tt && m_tt->texttype_fn_destroy) - m_tt->texttype_fn_destroy(m_tt); -} - bool TableMatcher::matchTable(const MetaName& tableName) { try @@ -104,18 +77,10 @@ bool TableMatcher::matchTable(const MetaName& tableName) enabled = true; if (m_includeMatcher) - { - m_includeMatcher->reset(); - m_includeMatcher->process((const UCHAR*) tableName.c_str(), tableName.length()); - enabled = m_includeMatcher->result(); - } + enabled = m_includeMatcher->matches(tableName.c_str(), tableName.length()); if (enabled && m_excludeMatcher) - { - m_excludeMatcher->reset(); - m_excludeMatcher->process((const UCHAR*) tableName.c_str(), tableName.length()); - enabled = !m_excludeMatcher->result(); - } + enabled = !m_excludeMatcher->matches(tableName.c_str(), tableName.length()); m_tables.put(tableName, enabled); } diff --git a/src/jrd/replication/Manager.h b/src/jrd/replication/Manager.h index d56f1b1cef..9b0232c785 100644 --- a/src/jrd/replication/Manager.h +++ b/src/jrd/replication/Manager.h @@ -26,9 +26,9 @@ #include "../common/classes/array.h" #include "../common/classes/semaphore.h" +#include "../common/SimilarToRegex.h" #include "../common/os/guid.h" #include "../common/isc_s_proto.h" -#include "../../jrd/SimilarToMatcher.h" #include "../../jrd/intl_classes.h" #include "Config.h" @@ -38,25 +38,18 @@ namespace Replication { class TableMatcher { - typedef Jrd::UpcaseConverter SimilarConverter; - typedef Firebird::SimilarToMatcher SimilarMatcher; typedef Firebird::GenericMap > > TablePermissionMap; public: TableMatcher(MemoryPool& pool, const Firebird::string& includeFilter, const Firebird::string& excludeFilter); - ~TableMatcher(); bool matchTable(const Firebird::MetaName& tableName); private: - charset* m_cs; - Firebird::AutoPtr m_tt; - Firebird::AutoPtr m_charSet; - Firebird::AutoPtr m_textType; - Firebird::AutoPtr m_includeMatcher; - Firebird::AutoPtr m_excludeMatcher; + Firebird::AutoPtr m_includeMatcher; + Firebird::AutoPtr m_excludeMatcher; TablePermissionMap m_tables; }; diff --git a/src/jrd/validation.cpp b/src/jrd/validation.cpp index f902a19e75..409f824d0e 100644 --- a/src/jrd/validation.cpp +++ b/src/jrd/validation.cpp @@ -570,7 +570,6 @@ VI. ADDITIONAL NOTES #include "../common/db_alias.h" #include "../jrd/intl_proto.h" #include "../jrd/lck_proto.h" -#include "../jrd/Collation.h" #ifdef DEBUG_VAL_VERBOSE #include "../jrd/dmp_proto.h" @@ -592,18 +591,21 @@ static void print_rhd(USHORT, const rhd*); #endif -static PatternMatcher* createPatternMatcher(thread_db* tdbb, const char* pattern) +static SimilarToRegex* createPatternMatcher(thread_db* tdbb, const char* pattern) { - PatternMatcher* matcher = NULL; + SimilarToRegex* matcher = NULL; try { if (pattern) { const int len = strlen(pattern); - Collation* obj = INTL_texttype_lookup(tdbb, CS_UTF8); - matcher = obj->createSimilarToMatcher(*tdbb->getDefaultPool(), - (const UCHAR*) pattern, len, (UCHAR*) "\\", 1); + //// TODO: Should this be different than trace and replication + //// and use case sensitive matcher? + matcher = FB_NEW_POOL(*tdbb->getDefaultPool()) SimilarToRegex( + *tdbb->getDefaultPool(), false, + pattern, len, + "\\", 1); } } catch (const Exception& ex) @@ -870,8 +872,6 @@ Validation::Validation(thread_db* tdbb, UtilSvc* uSvc) : vdr_page_bitmap = NULL; vdr_service = uSvc; - vdr_tab_incl = vdr_tab_excl = NULL; - vdr_idx_incl = vdr_idx_excl = NULL; vdr_lock_tout = -10; if (uSvc) { @@ -882,11 +882,6 @@ Validation::Validation(thread_db* tdbb, UtilSvc* uSvc) : Validation::~Validation() { - delete vdr_tab_incl; - delete vdr_tab_excl; - delete vdr_idx_incl; - delete vdr_idx_excl; - output("Validation finished\n"); } @@ -1654,22 +1649,14 @@ void Validation::walk_database() if (vdr_tab_incl) { - vdr_tab_incl->reset(); - if (!vdr_tab_incl->process((UCHAR*) relation->rel_name.c_str(), relation->rel_name.length()) || - !vdr_tab_incl->result()) - { + if (!vdr_tab_incl->matches(relation->rel_name.c_str(), relation->rel_name.length())) continue; - } } if (vdr_tab_excl) { - vdr_tab_excl->reset(); - if (!vdr_tab_excl->process((UCHAR*) relation->rel_name.c_str(), relation->rel_name.length()) || - vdr_tab_excl->result()) - { + if (vdr_tab_excl->matches(relation->rel_name.c_str(), relation->rel_name.length())) continue; - } } // We can't realiable track double allocated page's when validating online. @@ -3163,15 +3150,13 @@ Validation::RTN Validation::walk_root(jrd_rel* relation) if (vdr_idx_incl) { - vdr_idx_incl->reset(); - if (!vdr_idx_incl->process((UCHAR*) index.c_str(), index.length()) || !vdr_idx_incl->result()) + if (!vdr_idx_incl->matches(relation->rel_name.c_str(), relation->rel_name.length())) continue; } if (vdr_idx_excl) { - vdr_idx_excl->reset(); - if (!vdr_idx_excl->process((UCHAR*) index.c_str(), index.length()) || vdr_idx_excl->result()) + if (vdr_idx_excl->matches(relation->rel_name.c_str(), relation->rel_name.length())) continue; } diff --git a/src/jrd/validation.h b/src/jrd/validation.h index c45a6d5197..a6aa2c5279 100644 --- a/src/jrd/validation.h +++ b/src/jrd/validation.h @@ -28,6 +28,7 @@ #include "fb_types.h" #include "../common/classes/array.h" +#include "../common/SimilarToRegex.h" #include "../jrd/ods.h" #include "../jrd/cch.h" #include "../jrd/sbm.h" @@ -150,10 +151,10 @@ private: ULONG vdr_err_counts[VAL_MAX_ERROR]; Firebird::UtilSvc* vdr_service; - PatternMatcher* vdr_tab_incl; - PatternMatcher* vdr_tab_excl; - PatternMatcher* vdr_idx_incl; - PatternMatcher* vdr_idx_excl; + Firebird::AutoPtr vdr_tab_incl; + Firebird::AutoPtr vdr_tab_excl; + Firebird::AutoPtr vdr_idx_incl; + Firebird::AutoPtr vdr_idx_excl; int vdr_lock_tout; void checkDPinPP(jrd_rel *relation, SLONG page_number); void checkDPinPIP(jrd_rel *relation, SLONG page_number); diff --git a/src/utilities/CMakeLists.txt b/src/utilities/CMakeLists.txt index b275a1e7e3..2330750bae 100644 --- a/src/utilities/CMakeLists.txt +++ b/src/utilities/CMakeLists.txt @@ -46,7 +46,6 @@ set(fbtrace_src ntrace/TraceConfiguration.cpp ntrace/traceplugin.cpp ntrace/TracePluginImpl.cpp - ntrace/TraceUnicodeUtils.cpp ntrace/os/platform.h ) @@ -70,11 +69,11 @@ if (WIN32) set(instreg_src install/install_reg.cpp install/registry.cpp - + install/registry.h install/regis_proto.h ) - add_executable (instreg ${instreg_src} ${VERSION_RC}) + add_executable (instreg ${instreg_src} ${VERSION_RC}) ########################################################################### # EXECUTABLE instsvc @@ -86,7 +85,7 @@ if (WIN32) install/servi_proto.h ) add_executable (instsvc ${instsvc_src} ${VERSION_RC}) - target_link_libraries (instsvc common yvalve) + target_link_libraries (instsvc common yvalve) ########################################################################### # EXECUTABLE instclient diff --git a/src/utilities/ntrace/TraceConfiguration.cpp b/src/utilities/ntrace/TraceConfiguration.cpp index bdeb1d00bf..2c4136e1f3 100644 --- a/src/utilities/ntrace/TraceConfiguration.cpp +++ b/src/utilities/ntrace/TraceConfiguration.cpp @@ -26,9 +26,7 @@ */ #include "TraceConfiguration.h" -#include "TraceUnicodeUtils.h" -#include "../../jrd/evl_string.h" -#include "../../jrd/SimilarToMatcher.h" +#include "../../common/SimilarToRegex.h" #include "../../common/isc_f_proto.h" using namespace Firebird; @@ -67,26 +65,6 @@ void TraceCfgReader::readTraceConfiguration(const char* text, } -namespace -{ - template - class SystemToUtf8Converter : public PrevConverter - { - public: - SystemToUtf8Converter(MemoryPool& pool, Jrd::TextType* obj, const UCHAR*& str, SLONG& len) - : PrevConverter(pool, obj, str, len) - { - buffer.assign(reinterpret_cast(str), len); - ISC_systemToUtf8(buffer); - str = reinterpret_cast(buffer.c_str()); - len = buffer.length(); - } - - private: - string buffer; - }; -} - #define ERROR_PREFIX "error while parsing trace configuration\n\t" void TraceCfgReader::readConfig() @@ -156,31 +134,28 @@ void TraceCfgReader::readConfig() try { #ifdef WIN_NT // !CASE_SENSITIVITY - typedef Jrd::UpcaseConverter > SimilarConverter; + const bool caseInsensitive = true; #else - typedef SystemToUtf8Converter<> SimilarConverter; + const bool caseInsensitive = false; #endif + string utf8Pattern = pattern; + ISC_systemToUtf8(utf8Pattern); - UnicodeCollationHolder unicodeCollation(*getDefaultMemoryPool()); - Jrd::TextType* textType = unicodeCollation.getTextType(); - - SimilarToMatcher > matcher( - *getDefaultMemoryPool(), textType, (const UCHAR*) pattern.c_str(), - pattern.length(), '\\', true); + SimilarToRegex matcher(*getDefaultMemoryPool(), caseInsensitive, + utf8Pattern.c_str(), utf8Pattern.length(), "\\", 1); regExpOk = true; - matcher.process((const UCHAR*) m_databaseName.c_str(), m_databaseName.length()); - if (matcher.result()) - { - for (unsigned i = 0; - i <= matcher.getNumBranches() && i < FB_NELEM(m_subpatterns); ++i) - { - unsigned start, length; - matcher.getBranchInfo(i, &start, &length); + PathName utf8DatabaseName = m_databaseName; + ISC_systemToUtf8(utf8DatabaseName); + Array matchPosArray; - m_subpatterns[i].start = start; - m_subpatterns[i].end = start + length; + if (matcher.matches(utf8DatabaseName.c_str(), utf8DatabaseName.length(), &matchPosArray)) + { + for (unsigned i = 0; i < matchPosArray.getCount() && i < FB_NELEM(m_subpatterns); ++i) + { + m_subpatterns[i].start = matchPosArray[i].start; + m_subpatterns[i].end = matchPosArray[i].start + matchPosArray[i].length; } match = exactMatch = true; diff --git a/src/utilities/ntrace/TracePluginImpl.cpp b/src/utilities/ntrace/TracePluginImpl.cpp index d4793ebf81..214388638c 100644 --- a/src/utilities/ntrace/TracePluginImpl.cpp +++ b/src/utilities/ntrace/TracePluginImpl.cpp @@ -99,7 +99,6 @@ TracePluginImpl::TracePluginImpl(IPluginBase* plugin, transactions(getDefaultMemoryPool()), statements(getDefaultMemoryPool()), services(getDefaultMemoryPool()), - unicodeCollation(*getDefaultMemoryPool()), include_codes(*getDefaultMemoryPool()), exclude_codes(*getDefaultMemoryPool()) { @@ -124,8 +123,6 @@ TracePluginImpl::TracePluginImpl(IPluginBase* plugin, logWriter->addRef(); } - Jrd::TextType* textType = unicodeCollation.getTextType(); - // Compile filtering regular expressions const char* str = NULL; try @@ -136,9 +133,10 @@ TracePluginImpl::TracePluginImpl(IPluginBase* plugin, string filter(config.include_filter); ISC_systemToUtf8(filter); - include_matcher = FB_NEW TraceSimilarToMatcher( - *getDefaultMemoryPool(), textType, (const UCHAR*) filter.c_str(), - filter.length(), '\\', true); + include_matcher = FB_NEW SimilarToRegex( + *getDefaultMemoryPool(), true, + filter.c_str(), filter.length(), + "\\", 1); } if (config.exclude_filter.hasData()) @@ -147,9 +145,10 @@ TracePluginImpl::TracePluginImpl(IPluginBase* plugin, string filter(config.exclude_filter); ISC_systemToUtf8(filter); - exclude_matcher = FB_NEW TraceSimilarToMatcher( - *getDefaultMemoryPool(), textType, (const UCHAR*) filter.c_str(), - filter.length(), '\\', true); + exclude_matcher = FB_NEW SimilarToRegex( + *getDefaultMemoryPool(), true, + filter.c_str(), filter.length(), + "\\", 1); } } catch (const Exception&) @@ -1546,18 +1545,10 @@ void TracePluginImpl::register_sql_statement(ITraceSQLStatement* statement) return; if (config.include_filter.hasData()) - { - include_matcher->reset(); - include_matcher->process((const UCHAR*) sql, sql_length); - need_statement = include_matcher->result(); - } + need_statement = include_matcher->matches(sql, sql_length); if (need_statement && config.exclude_filter.hasData()) - { - exclude_matcher->reset(); - exclude_matcher->process((const UCHAR*) sql, sql_length); - need_statement = !exclude_matcher->result(); - } + need_statement = !exclude_matcher->matches(sql, sql_length); if (need_statement) { @@ -1949,18 +1940,10 @@ bool TracePluginImpl::checkServiceFilter(ITraceServiceConnection* service, bool bool enabled = true; if (config.include_filter.hasData()) - { - include_matcher->reset(); - include_matcher->process((const UCHAR*) svcName, svcNameLen); - enabled = include_matcher->result(); - } + enabled = include_matcher->matches(svcName, svcNameLen); if (enabled && config.exclude_filter.hasData()) - { - exclude_matcher->reset(); - exclude_matcher->process((const UCHAR*) svcName, svcNameLen); - enabled = !exclude_matcher->result(); - } + enabled = !exclude_matcher->matches(svcName, svcNameLen); if (data) { data->enabled = enabled; diff --git a/src/utilities/ntrace/TracePluginImpl.h b/src/utilities/ntrace/TracePluginImpl.h index 8824c3db4d..2c99be0d3d 100644 --- a/src/utilities/ntrace/TracePluginImpl.h +++ b/src/utilities/ntrace/TracePluginImpl.h @@ -32,11 +32,7 @@ #include "firebird.h" #include "../../jrd/ntrace.h" #include "TracePluginConfig.h" -#include "TraceUnicodeUtils.h" -#include "../../jrd/intl_classes.h" -#include "../../jrd/evl_string.h" -#include "../../common/TextType.h" -#include "../../jrd/SimilarToMatcher.h" +#include "../../common/SimilarToRegex.h" #include "../../common/classes/rwlock.h" #include "../../common/classes/GenericMap.h" #include "../../common/classes/locks.h" @@ -168,10 +164,7 @@ private: // Lock for log rotation Firebird::RWLock renameLock; - UnicodeCollationHolder unicodeCollation; - typedef Firebird::SimilarToMatcher > > - TraceSimilarToMatcher; - Firebird::AutoPtr include_matcher, exclude_matcher; + Firebird::AutoPtr include_matcher, exclude_matcher; // Filters for gds error codes typedef Firebird::SortedArray GdsCodesArray; diff --git a/src/utilities/ntrace/TraceUnicodeUtils.cpp b/src/utilities/ntrace/TraceUnicodeUtils.cpp deleted file mode 100644 index 80626af542..0000000000 --- a/src/utilities/ntrace/TraceUnicodeUtils.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * PROGRAM: Firebird Trace Services - * MODULE: TraceUnicodeUtils.cpp - * DESCRIPTION: Unicode support for trace needs - * - * The contents of this file are subject to the Initial - * Developer's Public License Version 1.0 (the "License"); - * you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. - * - * Software distributed under the License is distributed AS IS, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. - * See the License for the specific language governing rights - * and limitations under the License. - * - * The Original Code was created by Khorsun Vladyslav - * for the Firebird Open Source RDBMS project. - * - * Copyright (c) 2010 Khorsun Vladyslav - * and all contributors signed below. - * - * All Rights Reserved. - * Contributor(s): ______________________________________. - * Adriano dos Santos Fernandes - * - */ - - -#include "TraceUnicodeUtils.h" - -using namespace Firebird; - -UnicodeCollationHolder::UnicodeCollationHolder(MemoryPool& pool) -{ - cs = FB_NEW_POOL(pool) charset; - tt = FB_NEW_POOL(pool) texttype; - - IntlUtil::initUtf8Charset(cs); - - string collAttributes("ICU-VERSION="); - collAttributes += Jrd::UnicodeUtil::getDefaultIcuVersion(); - IntlUtil::setupIcuAttributes(cs, collAttributes, "", collAttributes); - - UCharBuffer collAttributesBuffer; - collAttributesBuffer.push(reinterpret_cast(collAttributes.c_str()), - collAttributes.length()); - - if (!IntlUtil::initUnicodeCollation(tt, cs, "UNICODE", 0, collAttributesBuffer, string())) - fatal_exception::raiseFmt("cannot initialize UNICODE collation to use in trace plugin"); - - charSet = Jrd::CharSet::createInstance(pool, 0, cs); - textType = FB_NEW_POOL(pool) Jrd::TextType(0, tt, charSet); -} - -UnicodeCollationHolder::~UnicodeCollationHolder() -{ - fb_assert(tt->texttype_fn_destroy); - - if (tt->texttype_fn_destroy) - tt->texttype_fn_destroy(tt); - - // cs should be deleted by texttype_fn_destroy call above - delete tt; -} diff --git a/src/utilities/ntrace/TraceUnicodeUtils.h b/src/utilities/ntrace/TraceUnicodeUtils.h deleted file mode 100644 index a66f428318..0000000000 --- a/src/utilities/ntrace/TraceUnicodeUtils.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * PROGRAM: Firebird Trace Services - * MODULE: TraceUnicodeUtils.h - * DESCRIPTION: Unicode support for trace needs - * - * The contents of this file are subject to the Initial - * Developer's Public License Version 1.0 (the "License"); - * you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. - * - * Software distributed under the License is distributed AS IS, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. - * See the License for the specific language governing rights - * and limitations under the License. - * - * The Original Code was created by Khorsun Vladyslav - * for the Firebird Open Source RDBMS project. - * - * Copyright (c) 2010 Khorsun Vladyslav - * and all contributors signed below. - * - * All Rights Reserved. - * Contributor(s): ______________________________________. - * - */ - -#ifndef TRACE_UNICODE_UTILS_H -#define TRACE_UNICODE_UTILS_H - -#include "firebird.h" -#include "../../common/classes/fb_string.h" -#include "../../jrd/intl_classes.h" -#include "../../common/TextType.h" -#include "../../common/unicode_util.h" - - -class UnicodeCollationHolder -{ -private: - charset* cs; - texttype* tt; - Firebird::AutoPtr charSet; - Firebird::AutoPtr textType; - -public: - explicit UnicodeCollationHolder(Firebird::MemoryPool& pool); - ~UnicodeCollationHolder(); - - Jrd::TextType* getTextType() - { - return textType; - } -}; - - -#endif // TRACE_UNICODE_UTILS_H From 65f003da0d951514fab1ab5d4f46455acb330c01 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sun, 11 Aug 2019 11:10:29 -0300 Subject: [PATCH 002/274] Import re2 version 2019-08-01. --- extern/re2/.gitignore | 5 + extern/re2/.travis.yml | 179 + extern/re2/AUTHORS | 13 + extern/re2/BUILD | 239 + extern/re2/CMakeLists.txt | 152 + extern/re2/CONTRIBUTING.md | 2 + extern/re2/CONTRIBUTORS | 41 + extern/re2/LICENSE | 27 + extern/re2/README | 38 + extern/re2/WORKSPACE | 6 + extern/re2/benchlog/benchplot.py | 98 + extern/re2/benchlog/mktable | 155 + extern/re2/doc/README.xkcd | 1 + extern/re2/doc/mksyntaxgo | 41 + extern/re2/doc/mksyntaxhtml | 42 + extern/re2/doc/mksyntaxwiki | 36 + extern/re2/doc/syntax.html | 409 ++ extern/re2/doc/syntax.txt | 452 ++ extern/re2/doc/xkcd.png | Bin 0 -> 26496 bytes extern/re2/kokoro/bazel.sh | 26 + extern/re2/kokoro/cmake.sh | 25 + extern/re2/kokoro/macos-bazel.cfg | 1 + extern/re2/kokoro/macos-bazel.sh | 4 + extern/re2/kokoro/macos-cmake.cfg | 1 + extern/re2/kokoro/macos-cmake.sh | 4 + extern/re2/kokoro/ubuntu-bazel.cfg | 1 + extern/re2/kokoro/ubuntu-bazel.sh | 4 + extern/re2/kokoro/windows-bazel.bat | 2 + extern/re2/kokoro/windows-bazel.cfg | 1 + extern/re2/kokoro/windows-cmake.bat | 2 + extern/re2/kokoro/windows-cmake.cfg | 1 + extern/re2/lib/git/commit-msg.hook | 104 + extern/re2/libre2.symbols | 16 + extern/re2/libre2.symbols.darwin | 12 + extern/re2/re2.pc | 10 + extern/re2/re2/bitmap256.h | 118 + extern/re2/re2/bitstate.cc | 378 + extern/re2/re2/compile.cc | 1279 ++++ extern/re2/re2/dfa.cc | 2151 ++++++ extern/re2/re2/filtered_re2.cc | 121 + extern/re2/re2/filtered_re2.h | 109 + extern/re2/re2/fuzzing/re2_fuzzer.cc | 173 + extern/re2/re2/make_perl_groups.pl | 116 + extern/re2/re2/make_unicode_casefold.py | 151 + extern/re2/re2/make_unicode_groups.py | 117 + extern/re2/re2/mimics_pcre.cc | 187 + extern/re2/re2/nfa.cc | 757 ++ extern/re2/re2/onepass.cc | 623 ++ extern/re2/re2/parse.cc | 2463 +++++++ extern/re2/re2/perl_groups.cc | 119 + extern/re2/re2/prefilter.cc | 710 ++ extern/re2/re2/prefilter.h | 108 + extern/re2/re2/prefilter_tree.cc | 407 ++ extern/re2/re2/prefilter_tree.h | 139 + extern/re2/re2/prog.cc | 921 +++ extern/re2/re2/prog.h | 432 ++ extern/re2/re2/re2.cc | 1236 ++++ extern/re2/re2/re2.h | 959 +++ extern/re2/re2/regexp.cc | 971 +++ extern/re2/re2/regexp.h | 652 ++ extern/re2/re2/set.cc | 153 + extern/re2/re2/set.h | 80 + extern/re2/re2/simplify.cc | 655 ++ extern/re2/re2/stringpiece.cc | 65 + extern/re2/re2/stringpiece.h | 210 + extern/re2/re2/testing/backtrack.cc | 273 + extern/re2/re2/testing/charclass_test.cc | 226 + extern/re2/re2/testing/compile_test.cc | 397 ++ extern/re2/re2/testing/dfa_test.cc | 381 + extern/re2/re2/testing/dump.cc | 169 + extern/re2/re2/testing/exhaustive1_test.cc | 44 + extern/re2/re2/testing/exhaustive2_test.cc | 73 + extern/re2/re2/testing/exhaustive3_test.cc | 100 + extern/re2/re2/testing/exhaustive_test.cc | 36 + extern/re2/re2/testing/exhaustive_tester.cc | 188 + extern/re2/re2/testing/exhaustive_tester.h | 105 + extern/re2/re2/testing/filtered_re2_test.cc | 294 + extern/re2/re2/testing/mimics_pcre_test.cc | 77 + extern/re2/re2/testing/null_walker.cc | 46 + extern/re2/re2/testing/parse_test.cc | 508 ++ extern/re2/re2/testing/possible_match_test.cc | 247 + extern/re2/re2/testing/random_test.cc | 99 + extern/re2/re2/testing/re2_arg_test.cc | 135 + extern/re2/re2/testing/re2_test.cc | 1631 +++++ extern/re2/re2/testing/regexp_benchmark.cc | 1586 +++++ extern/re2/re2/testing/regexp_generator.cc | 276 + extern/re2/re2/testing/regexp_generator.h | 77 + extern/re2/re2/testing/regexp_test.cc | 86 + .../re2/re2/testing/required_prefix_test.cc | 72 + extern/re2/re2/testing/search_test.cc | 332 + extern/re2/re2/testing/set_test.cc | 204 + extern/re2/re2/testing/simplify_test.cc | 273 + extern/re2/re2/testing/string_generator.cc | 114 + extern/re2/re2/testing/string_generator.h | 63 + .../re2/re2/testing/string_generator_test.cc | 110 + extern/re2/re2/testing/tester.cc | 669 ++ extern/re2/re2/testing/tester.h | 123 + extern/re2/re2/tostring.cc | 351 + extern/re2/re2/unicode.py | 303 + extern/re2/re2/unicode_casefold.cc | 578 ++ extern/re2/re2/unicode_casefold.h | 78 + extern/re2/re2/unicode_groups.cc | 6162 +++++++++++++++++ extern/re2/re2/unicode_groups.h | 67 + extern/re2/re2/walker-inl.h | 248 + extern/re2/re2_test.bzl | 12 + extern/re2/runtests | 33 + extern/re2/testinstall.cc | 24 + extern/re2/util/benchmark.cc | 161 + extern/re2/util/benchmark.h | 43 + extern/re2/util/flags.h | 29 + extern/re2/util/fuzz.cc | 21 + extern/re2/util/logging.h | 109 + extern/re2/util/mix.h | 41 + extern/re2/util/mutex.h | 131 + extern/re2/util/pcre.cc | 1051 +++ extern/re2/util/pcre.h | 681 ++ extern/re2/util/pod_array.h | 55 + extern/re2/util/rune.cc | 260 + extern/re2/util/sparse_array.h | 392 ++ extern/re2/util/sparse_set.h | 264 + extern/re2/util/strutil.cc | 149 + extern/re2/util/strutil.h | 21 + extern/re2/util/test.cc | 31 + extern/re2/util/test.h | 58 + extern/re2/util/utf.h | 44 + extern/re2/util/util.h | 34 + 126 files changed, 39155 insertions(+) create mode 100644 extern/re2/.gitignore create mode 100644 extern/re2/.travis.yml create mode 100644 extern/re2/AUTHORS create mode 100644 extern/re2/BUILD create mode 100644 extern/re2/CMakeLists.txt create mode 100644 extern/re2/CONTRIBUTING.md create mode 100644 extern/re2/CONTRIBUTORS create mode 100644 extern/re2/LICENSE create mode 100644 extern/re2/README create mode 100644 extern/re2/WORKSPACE create mode 100644 extern/re2/benchlog/benchplot.py create mode 100644 extern/re2/benchlog/mktable create mode 100644 extern/re2/doc/README.xkcd create mode 100644 extern/re2/doc/mksyntaxgo create mode 100644 extern/re2/doc/mksyntaxhtml create mode 100644 extern/re2/doc/mksyntaxwiki create mode 100644 extern/re2/doc/syntax.html create mode 100644 extern/re2/doc/syntax.txt create mode 100644 extern/re2/doc/xkcd.png create mode 100644 extern/re2/kokoro/bazel.sh create mode 100644 extern/re2/kokoro/cmake.sh create mode 100644 extern/re2/kokoro/macos-bazel.cfg create mode 100644 extern/re2/kokoro/macos-bazel.sh create mode 100644 extern/re2/kokoro/macos-cmake.cfg create mode 100644 extern/re2/kokoro/macos-cmake.sh create mode 100644 extern/re2/kokoro/ubuntu-bazel.cfg create mode 100644 extern/re2/kokoro/ubuntu-bazel.sh create mode 100644 extern/re2/kokoro/windows-bazel.bat create mode 100644 extern/re2/kokoro/windows-bazel.cfg create mode 100644 extern/re2/kokoro/windows-cmake.bat create mode 100644 extern/re2/kokoro/windows-cmake.cfg create mode 100644 extern/re2/lib/git/commit-msg.hook create mode 100644 extern/re2/libre2.symbols create mode 100644 extern/re2/libre2.symbols.darwin create mode 100644 extern/re2/re2.pc create mode 100644 extern/re2/re2/bitmap256.h create mode 100644 extern/re2/re2/bitstate.cc create mode 100644 extern/re2/re2/compile.cc create mode 100644 extern/re2/re2/dfa.cc create mode 100644 extern/re2/re2/filtered_re2.cc create mode 100644 extern/re2/re2/filtered_re2.h create mode 100644 extern/re2/re2/fuzzing/re2_fuzzer.cc create mode 100644 extern/re2/re2/make_perl_groups.pl create mode 100644 extern/re2/re2/make_unicode_casefold.py create mode 100644 extern/re2/re2/make_unicode_groups.py create mode 100644 extern/re2/re2/mimics_pcre.cc create mode 100644 extern/re2/re2/nfa.cc create mode 100644 extern/re2/re2/onepass.cc create mode 100644 extern/re2/re2/parse.cc create mode 100644 extern/re2/re2/perl_groups.cc create mode 100644 extern/re2/re2/prefilter.cc create mode 100644 extern/re2/re2/prefilter.h create mode 100644 extern/re2/re2/prefilter_tree.cc create mode 100644 extern/re2/re2/prefilter_tree.h create mode 100644 extern/re2/re2/prog.cc create mode 100644 extern/re2/re2/prog.h create mode 100644 extern/re2/re2/re2.cc create mode 100644 extern/re2/re2/re2.h create mode 100644 extern/re2/re2/regexp.cc create mode 100644 extern/re2/re2/regexp.h create mode 100644 extern/re2/re2/set.cc create mode 100644 extern/re2/re2/set.h create mode 100644 extern/re2/re2/simplify.cc create mode 100644 extern/re2/re2/stringpiece.cc create mode 100644 extern/re2/re2/stringpiece.h create mode 100644 extern/re2/re2/testing/backtrack.cc create mode 100644 extern/re2/re2/testing/charclass_test.cc create mode 100644 extern/re2/re2/testing/compile_test.cc create mode 100644 extern/re2/re2/testing/dfa_test.cc create mode 100644 extern/re2/re2/testing/dump.cc create mode 100644 extern/re2/re2/testing/exhaustive1_test.cc create mode 100644 extern/re2/re2/testing/exhaustive2_test.cc create mode 100644 extern/re2/re2/testing/exhaustive3_test.cc create mode 100644 extern/re2/re2/testing/exhaustive_test.cc create mode 100644 extern/re2/re2/testing/exhaustive_tester.cc create mode 100644 extern/re2/re2/testing/exhaustive_tester.h create mode 100644 extern/re2/re2/testing/filtered_re2_test.cc create mode 100644 extern/re2/re2/testing/mimics_pcre_test.cc create mode 100644 extern/re2/re2/testing/null_walker.cc create mode 100644 extern/re2/re2/testing/parse_test.cc create mode 100644 extern/re2/re2/testing/possible_match_test.cc create mode 100644 extern/re2/re2/testing/random_test.cc create mode 100644 extern/re2/re2/testing/re2_arg_test.cc create mode 100644 extern/re2/re2/testing/re2_test.cc create mode 100644 extern/re2/re2/testing/regexp_benchmark.cc create mode 100644 extern/re2/re2/testing/regexp_generator.cc create mode 100644 extern/re2/re2/testing/regexp_generator.h create mode 100644 extern/re2/re2/testing/regexp_test.cc create mode 100644 extern/re2/re2/testing/required_prefix_test.cc create mode 100644 extern/re2/re2/testing/search_test.cc create mode 100644 extern/re2/re2/testing/set_test.cc create mode 100644 extern/re2/re2/testing/simplify_test.cc create mode 100644 extern/re2/re2/testing/string_generator.cc create mode 100644 extern/re2/re2/testing/string_generator.h create mode 100644 extern/re2/re2/testing/string_generator_test.cc create mode 100644 extern/re2/re2/testing/tester.cc create mode 100644 extern/re2/re2/testing/tester.h create mode 100644 extern/re2/re2/tostring.cc create mode 100644 extern/re2/re2/unicode.py create mode 100644 extern/re2/re2/unicode_casefold.cc create mode 100644 extern/re2/re2/unicode_casefold.h create mode 100644 extern/re2/re2/unicode_groups.cc create mode 100644 extern/re2/re2/unicode_groups.h create mode 100644 extern/re2/re2/walker-inl.h create mode 100644 extern/re2/re2_test.bzl create mode 100644 extern/re2/runtests create mode 100644 extern/re2/testinstall.cc create mode 100644 extern/re2/util/benchmark.cc create mode 100644 extern/re2/util/benchmark.h create mode 100644 extern/re2/util/flags.h create mode 100644 extern/re2/util/fuzz.cc create mode 100644 extern/re2/util/logging.h create mode 100644 extern/re2/util/mix.h create mode 100644 extern/re2/util/mutex.h create mode 100644 extern/re2/util/pcre.cc create mode 100644 extern/re2/util/pcre.h create mode 100644 extern/re2/util/pod_array.h create mode 100644 extern/re2/util/rune.cc create mode 100644 extern/re2/util/sparse_array.h create mode 100644 extern/re2/util/sparse_set.h create mode 100644 extern/re2/util/strutil.cc create mode 100644 extern/re2/util/strutil.h create mode 100644 extern/re2/util/test.cc create mode 100644 extern/re2/util/test.h create mode 100644 extern/re2/util/utf.h create mode 100644 extern/re2/util/util.h diff --git a/extern/re2/.gitignore b/extern/re2/.gitignore new file mode 100644 index 0000000000..a671fe2cf0 --- /dev/null +++ b/extern/re2/.gitignore @@ -0,0 +1,5 @@ +*.pyc +*.orig +core +obj/ +benchlog.* diff --git a/extern/re2/.travis.yml b/extern/re2/.travis.yml new file mode 100644 index 0000000000..f89c96d825 --- /dev/null +++ b/extern/re2/.travis.yml @@ -0,0 +1,179 @@ +language: cpp +sudo: false +dist: trusty +script: + - make + - make test +matrix: + include: + + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-4.8 + env: + - MATRIX_EVAL="CC=gcc-4.8 CXX=g++-4.8" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-4.9 + env: + - MATRIX_EVAL="CC=gcc-4.9 CXX=g++-4.9" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-5 + env: + - MATRIX_EVAL="CC=gcc-5 CXX=g++-5" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-6 + env: + - MATRIX_EVAL="CC=gcc-6 CXX=g++-6" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-7 + env: + - MATRIX_EVAL="CC=gcc-7 CXX=g++-7" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-8 + env: + - MATRIX_EVAL="CC=gcc-8 CXX=g++-8" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-9 + env: + - MATRIX_EVAL="CC=gcc-9 CXX=g++-9" + + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.5 + packages: + - clang-3.5 + env: + - MATRIX_EVAL="CC=clang-3.5 CXX=clang++-3.5" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.6 + packages: + - clang-3.6 + env: + - MATRIX_EVAL="CC=clang-3.6 CXX=clang++-3.6" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.7 + packages: + - clang-3.7 + env: + - MATRIX_EVAL="CC=clang-3.7 CXX=clang++-3.7" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.8 + packages: + - clang-3.8 + env: + - MATRIX_EVAL="CC=clang-3.8 CXX=clang++-3.8" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.9 + packages: + - clang-3.9 + env: + - MATRIX_EVAL="CC=clang-3.9 CXX=clang++-3.9" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-trusty-4.0 + packages: + - clang-4.0 + env: + - MATRIX_EVAL="CC=clang-4.0 CXX=clang++-4.0" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-trusty-5.0 + packages: + - clang-5.0 + env: + - MATRIX_EVAL="CC=clang-5.0 CXX=clang++-5.0" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - sourceline: 'deb https://apt.llvm.org/trusty/ llvm-toolchain-trusty-6.0 main' + key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' + packages: + - clang-6.0 + env: + - MATRIX_EVAL="CC=clang-6.0 CXX=clang++-6.0" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - sourceline: 'deb https://apt.llvm.org/trusty/ llvm-toolchain-trusty-7 main' + key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' + packages: + - clang-7 + env: + - MATRIX_EVAL="CC=clang-7 CXX=clang++-7" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - sourceline: 'deb https://apt.llvm.org/trusty/ llvm-toolchain-trusty-8 main' + key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' + packages: + - clang-8 + env: + - MATRIX_EVAL="CC=clang-8 CXX=clang++-8" + +before_install: + - eval "${MATRIX_EVAL}" diff --git a/extern/re2/AUTHORS b/extern/re2/AUTHORS new file mode 100644 index 0000000000..0754006fec --- /dev/null +++ b/extern/re2/AUTHORS @@ -0,0 +1,13 @@ +# This is the official list of RE2 authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# Please keep the list sorted. + +Google Inc. +Samsung Electronics +Stefano Rivera diff --git a/extern/re2/BUILD b/extern/re2/BUILD new file mode 100644 index 0000000000..30ce32094f --- /dev/null +++ b/extern/re2/BUILD @@ -0,0 +1,239 @@ +# Copyright 2009 The RE2 Authors. All Rights Reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Bazel (http://bazel.io/) BUILD file for RE2. + +licenses(["notice"]) + +exports_files(["LICENSE"]) + +config_setting( + name = "darwin", + values = {"cpu": "darwin"}, +) + +config_setting( + name = "windows", + values = {"cpu": "x64_windows"}, +) + +config_setting( + name = "windows_msvc", + values = {"cpu": "x64_windows_msvc"}, +) + +cc_library( + name = "re2", + srcs = [ + "re2/bitmap256.h", + "re2/bitstate.cc", + "re2/compile.cc", + "re2/dfa.cc", + "re2/filtered_re2.cc", + "re2/mimics_pcre.cc", + "re2/nfa.cc", + "re2/onepass.cc", + "re2/parse.cc", + "re2/perl_groups.cc", + "re2/prefilter.cc", + "re2/prefilter.h", + "re2/prefilter_tree.cc", + "re2/prefilter_tree.h", + "re2/prog.cc", + "re2/prog.h", + "re2/re2.cc", + "re2/regexp.cc", + "re2/regexp.h", + "re2/set.cc", + "re2/simplify.cc", + "re2/stringpiece.cc", + "re2/tostring.cc", + "re2/unicode_casefold.cc", + "re2/unicode_casefold.h", + "re2/unicode_groups.cc", + "re2/unicode_groups.h", + "re2/walker-inl.h", + "util/flags.h", + "util/logging.h", + "util/mix.h", + "util/mutex.h", + "util/pod_array.h", + "util/rune.cc", + "util/sparse_array.h", + "util/sparse_set.h", + "util/strutil.cc", + "util/strutil.h", + "util/utf.h", + "util/util.h", + ], + hdrs = [ + "re2/filtered_re2.h", + "re2/re2.h", + "re2/set.h", + "re2/stringpiece.h", + ], + copts = select({ + ":windows": [], + ":windows_msvc": [], + "//conditions:default": ["-pthread"], + }), + linkopts = select({ + # Darwin doesn't need `-pthread' when linking and it appears that + # older versions of Clang will warn about the unused command line + # argument, so just don't pass it. + ":darwin": [], + ":windows": [], + ":windows_msvc": [], + "//conditions:default": ["-pthread"], + }), + visibility = ["//visibility:public"], +) + +cc_library( + name = "testing", + testonly = 1, + srcs = [ + "re2/testing/backtrack.cc", + "re2/testing/dump.cc", + "re2/testing/exhaustive_tester.cc", + "re2/testing/null_walker.cc", + "re2/testing/regexp_generator.cc", + "re2/testing/string_generator.cc", + "re2/testing/tester.cc", + "util/pcre.cc", + ], + hdrs = [ + "re2/testing/exhaustive_tester.h", + "re2/testing/regexp_generator.h", + "re2/testing/string_generator.h", + "re2/testing/tester.h", + "util/benchmark.h", + "util/pcre.h", + "util/test.h", + ], + deps = [":re2"], +) + +cc_library( + name = "test", + testonly = 1, + srcs = ["util/test.cc"], + deps = [":testing"], +) + +load(":re2_test.bzl", "re2_test") + +re2_test( + "charclass_test", + size = "small", +) + +re2_test( + "compile_test", + size = "small", +) + +re2_test( + "filtered_re2_test", + size = "small", +) + +re2_test( + "mimics_pcre_test", + size = "small", +) + +re2_test( + "parse_test", + size = "small", +) + +re2_test( + "possible_match_test", + size = "small", +) + +re2_test( + "re2_arg_test", + size = "small", +) + +re2_test( + "re2_test", + size = "small", +) + +re2_test( + "regexp_test", + size = "small", +) + +re2_test( + "required_prefix_test", + size = "small", +) + +re2_test( + "search_test", + size = "small", +) + +re2_test( + "set_test", + size = "small", +) + +re2_test( + "simplify_test", + size = "small", +) + +re2_test( + "string_generator_test", + size = "small", +) + +re2_test( + "dfa_test", + size = "large", +) + +re2_test( + "exhaustive1_test", + size = "large", +) + +re2_test( + "exhaustive2_test", + size = "large", +) + +re2_test( + "exhaustive3_test", + size = "large", +) + +re2_test( + "exhaustive_test", + size = "large", +) + +re2_test( + "random_test", + size = "large", +) + +cc_library( + name = "benchmark", + testonly = 1, + srcs = ["util/benchmark.cc"], + deps = [":testing"], +) + +cc_binary( + name = "regexp_benchmark", + testonly = 1, + srcs = ["re2/testing/regexp_benchmark.cc"], + deps = [":benchmark"], +) diff --git a/extern/re2/CMakeLists.txt b/extern/re2/CMakeLists.txt new file mode 100644 index 0000000000..639c715a95 --- /dev/null +++ b/extern/re2/CMakeLists.txt @@ -0,0 +1,152 @@ +# Copyright 2015 The RE2 Authors. All Rights Reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Old enough to support Ubuntu Trusty. +cmake_minimum_required(VERSION 2.8.12) + +if(POLICY CMP0048) + cmake_policy(SET CMP0048 NEW) +endif() + +project(RE2 CXX) +include(CTest) + +option(BUILD_SHARED_LIBS "build shared libraries" OFF) +option(USEPCRE "use PCRE in tests and benchmarks" OFF) + +# CMake seems to have no way to enable/disable testing per subproject, +# so we provide an option similar to BUILD_TESTING, but just for RE2. +option(RE2_BUILD_TESTING "enable testing for RE2" ON) + +set(EXTRA_TARGET_LINK_LIBRARIES) + +if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + if(MSVC_VERSION LESS 1900) + message(FATAL_ERROR "you need Visual Studio 2015 or later") + endif() + if(BUILD_SHARED_LIBS) + # See http://www.kitware.com/blog/home/post/939 for details. + cmake_minimum_required(VERSION 3.4) + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) + endif() + # CMake defaults to /W3, but some users like /W4 (or /Wall) and /WX, + # so we disable various warnings that aren't particularly helpful. + add_compile_options(/wd4100 /wd4201 /wd4456 /wd4457 /wd4702 /wd4815) + # Without a byte order mark (BOM), Visual Studio assumes that the source + # file is encoded using the current user code page, so we specify UTF-8. + add_compile_options(/utf-8) +elseif(CYGWIN OR MINGW) + # See https://stackoverflow.com/questions/38139631 for details. + add_compile_options(-std=gnu++11) +elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + add_compile_options(-std=c++11) +endif() + +if(WIN32) + add_definitions(-DUNICODE -D_UNICODE -DSTRICT -DNOMINMAX) + add_definitions(-D_CRT_SECURE_NO_WARNINGS -D_SCL_SECURE_NO_WARNINGS) +elseif(UNIX) + add_compile_options(-pthread) + list(APPEND EXTRA_TARGET_LINK_LIBRARIES -pthread) +endif() + +if(USEPCRE) + add_definitions(-DUSEPCRE) + list(APPEND EXTRA_TARGET_LINK_LIBRARIES pcre) +endif() + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +set(RE2_SOURCES + re2/bitstate.cc + re2/compile.cc + re2/dfa.cc + re2/filtered_re2.cc + re2/mimics_pcre.cc + re2/nfa.cc + re2/onepass.cc + re2/parse.cc + re2/perl_groups.cc + re2/prefilter.cc + re2/prefilter_tree.cc + re2/prog.cc + re2/re2.cc + re2/regexp.cc + re2/set.cc + re2/simplify.cc + re2/stringpiece.cc + re2/tostring.cc + re2/unicode_casefold.cc + re2/unicode_groups.cc + util/rune.cc + util/strutil.cc + ) + +add_library(re2 ${RE2_SOURCES}) +add_library(re2::re2 ALIAS re2) + +if(RE2_BUILD_TESTING) + set(TESTING_SOURCES + re2/testing/backtrack.cc + re2/testing/dump.cc + re2/testing/exhaustive_tester.cc + re2/testing/null_walker.cc + re2/testing/regexp_generator.cc + re2/testing/string_generator.cc + re2/testing/tester.cc + util/pcre.cc + ) + + add_library(testing STATIC ${TESTING_SOURCES}) + + set(TEST_TARGETS + charclass_test + compile_test + filtered_re2_test + mimics_pcre_test + parse_test + possible_match_test + re2_test + re2_arg_test + regexp_test + required_prefix_test + search_test + set_test + simplify_test + string_generator_test + + dfa_test + exhaustive1_test + exhaustive2_test + exhaustive3_test + exhaustive_test + random_test + ) + + set(BENCHMARK_TARGETS + regexp_benchmark + ) + + foreach(target ${TEST_TARGETS}) + add_executable(${target} re2/testing/${target}.cc util/test.cc) + target_link_libraries(${target} testing re2 ${EXTRA_TARGET_LINK_LIBRARIES}) + add_test(NAME ${target} COMMAND ${target}) + endforeach(target) + + foreach(target ${BENCHMARK_TARGETS}) + add_executable(${target} re2/testing/${target}.cc util/benchmark.cc) + target_link_libraries(${target} testing re2 ${EXTRA_TARGET_LINK_LIBRARIES}) + endforeach(target) +endif() + +set(RE2_HEADERS + re2/filtered_re2.h + re2/re2.h + re2/set.h + re2/stringpiece.h + ) + +install(FILES ${RE2_HEADERS} DESTINATION include/re2) +install(TARGETS re2 EXPORT re2Config ARCHIVE DESTINATION lib LIBRARY DESTINATION lib RUNTIME DESTINATION bin INCLUDES DESTINATION include) +install(EXPORT re2Config DESTINATION lib/cmake/re2 NAMESPACE re2::) diff --git a/extern/re2/CONTRIBUTING.md b/extern/re2/CONTRIBUTING.md new file mode 100644 index 0000000000..3af2b0a76f --- /dev/null +++ b/extern/re2/CONTRIBUTING.md @@ -0,0 +1,2 @@ +RE2 uses Gerrit instead of GitHub pull requests. +See the [Contributing](https://github.com/google/re2/wiki/Contribute) wiki page. diff --git a/extern/re2/CONTRIBUTORS b/extern/re2/CONTRIBUTORS new file mode 100644 index 0000000000..1a1c84827d --- /dev/null +++ b/extern/re2/CONTRIBUTORS @@ -0,0 +1,41 @@ +# This is the official list of people who can contribute +# (and typically have contributed) code to the RE2 repository. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# The submission process automatically checks to make sure +# that people submitting code are listed in this file (by email address). +# +# Names should be added to this file only after verifying that +# the individual or the individual's organization has agreed to +# the appropriate Contributor License Agreement, found here: +# +# http://code.google.com/legal/individual-cla-v1.0.html +# http://code.google.com/legal/corporate-cla-v1.0.html +# +# The agreement for individuals can be filled out on the web. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file, depending on whether the +# individual or corporate CLA was used. + +# Names should be added to this file like so: +# Name + +# Please keep the list sorted. + +Dominic Battré +Doug Kwan +Dmitriy Vyukov +John Millikin +Mike Nazarewicz +Nico Weber +Pawel Hajdan +Rob Pike +Russ Cox +Sanjay Ghemawat +Stefano Rivera +Srinivasan Venkatachary +Viatcheslav Ostapenko diff --git a/extern/re2/LICENSE b/extern/re2/LICENSE new file mode 100644 index 0000000000..09e5ec1c74 --- /dev/null +++ b/extern/re2/LICENSE @@ -0,0 +1,27 @@ +// Copyright (c) 2009 The RE2 Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/extern/re2/README b/extern/re2/README new file mode 100644 index 0000000000..d1ef431b2b --- /dev/null +++ b/extern/re2/README @@ -0,0 +1,38 @@ +This is the source code repository for RE2, a regular expression library. + +For documentation about how to install and use RE2, +visit https://github.com/google/re2/. + +The short version is: + +make +make test +make install +make testinstall + +There is a fair amount of documentation (including code snippets) in +the re2.h header file. + +More information can be found on the wiki: +https://github.com/google/re2/wiki + +Issue tracker: +https://github.com/google/re2/issues + +Mailing list: +https://groups.google.com/group/re2-dev + +Unless otherwise noted, the RE2 source files are distributed +under the BSD-style license found in the LICENSE file. + +RE2's native language is C++. + +A C wrapper is at https://github.com/marcomaggi/cre2/. +An Erlang wrapper is at https://github.com/dukesoferl/re2/ and on Hex (hex.pm). +An Inferno wrapper is at https://github.com/powerman/inferno-re2/. +A Node.js wrapper is at https://github.com/uhop/node-re2/ and on NPM (npmjs.com). +An OCaml wrapper is at https://github.com/janestreet/re2/ and on OPAM (opam.ocaml.org). +A Perl wrapper is at https://github.com/dgl/re-engine-RE2/ and on CPAN (cpan.org). +A Python wrapper is at https://github.com/facebook/pyre2/ and on PyPI (pypi.org). +An R wrapper is at https://github.com/qinwf/re2r/ and on CRAN (cran.r-project.org). +A Ruby wrapper is at https://github.com/mudge/re2/ and on RubyGems (rubygems.org). diff --git a/extern/re2/WORKSPACE b/extern/re2/WORKSPACE new file mode 100644 index 0000000000..de481fe836 --- /dev/null +++ b/extern/re2/WORKSPACE @@ -0,0 +1,6 @@ +# Copyright 2009 The RE2 Authors. All Rights Reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Bazel (http://bazel.io/) WORKSPACE file for RE2. +workspace(name = "com_googlesource_code_re2") diff --git a/extern/re2/benchlog/benchplot.py b/extern/re2/benchlog/benchplot.py new file mode 100644 index 0000000000..104abe8e9e --- /dev/null +++ b/extern/re2/benchlog/benchplot.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python + +import argparse # for ArgumentParser +import subprocess # for Popen +import tempfile # for NamedTemporaryFile +import os # for remove + +class gnuplot(object): + + output = "result.png" + + script = """ + set terminal png size 1024, 768 + set output "{}.png" + set title "re2 benchlog" + set datafile separator ";" + set grid x y + set ylabel "MB/s" + set autoscale + plot """ + + template = """'{}' using 1:5:xticlabels(2) with linespoints linewidth 3 title "{}",\\\n""" + + benchdata = dict() + tempfiles = [] + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + """ + remove all temporary files + """ + + for filename in self.tempfiles: + os.remove(filename) + + def parse_re2_benchlog(self, filename): + """ + parse the input benchlog and return a dictionary contain bench data + """ + + benchdata = self.benchdata + + with open(filename) as f: + + for raw in f.readlines(): + + data = raw.split('\t') + + if len(data) == 4: + + data = data[0].split('/') + data[1:] + data = list(map(str.strip, data)) + + if not benchdata.get(data[0]): + benchdata[data[0]] = [ data[1:] ] + else: + benchdata[data[0]].append(data[1:]) + + def gen_csv(self): + """ + generate temporary csv files + """ + + for name, data in self.benchdata.items(): + + with tempfile.NamedTemporaryFile(delete=False) as f: + + for index, line in enumerate(data): + f.write('{};{}\n'.format(index, ';'.join(line)).encode()) + + self.tempfiles.append(f.name) + self.script = self.script + self.template.format(f.name, name) + + def run(self): + self.gen_csv() + script = self.script[:-3].format(self.output) + command = subprocess.Popen(['gnuplot'], stdin=subprocess.PIPE) + command.communicate(script.encode()) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='generate plots for benchlog') + parser.add_argument('benchlog', type=str, help='benchlog generated by re2') + args = parser.parse_args() + + try: + subprocess.Popen(['gnuplot'], stdin=subprocess.PIPE) + except FileNotFoundError: + print('you can install "gnuplot" to generate plots automatically') + exit(1) + + with gnuplot() as plot: + plot.output = args.benchlog + plot.parse_re2_benchlog(args.benchlog) + plot.run() diff --git a/extern/re2/benchlog/mktable b/extern/re2/benchlog/mktable new file mode 100644 index 0000000000..da0659820c --- /dev/null +++ b/extern/re2/benchlog/mktable @@ -0,0 +1,155 @@ +#!/usr/bin/perl +# XXX + +sub table() { + my ($name) = @_; + print <<'EOF'; + + +EOF + foreach my $sys (@sys) { + my $ns_pcre = $data{$sys}->{sprintf($name, "PCRE")}->{'ns/op'}; + my $ns_re2 = $data{$sys}->{sprintf($name, "RE2")}->{'ns/op'}; + printf "\n", $sysname{$sys}, $ns_pcre/1000., $ns_re2/1000.; + } + print <<'EOF'; + +
SystemPCRERE2
%s%.1f µs%.1f µs
+EOF +} + +@sizes = ( + "8", "16", "32", "64", "128", "256", "512", + "1K", "2K", "4K", "8K", "16K", "32K", "64K", "128K", "256K", "512K", + "1M", "2M", "4M", "8M", "16M" +); + +%color = ( + "PCRE" => "0.7 0 0", + "RE2" => "0 0 1", +); + +$ngraph = 0; + +sub graph() { + my ($name) = @_; + + my $sys = "wreck"; + my $base = sprintf("regexp3g%d", ++$ngraph); + + open(JGR, ">$base.jgr") || die "open >$base.jgr: $!"; + printf JGR "bbox -20 -12 392 95\n"; + printf JGR "newgraph clip x_translate 0.25 y_translate 0.25\n"; + $ymax = 0; + %lastx = (); + %lasty = (); + foreach my $who ("PCRE", "RE2") { + printf JGR "newcurve pts\n"; + for(my $i=0; $i<@sizes; $i++) { + my $key = sprintf("%s%s/%s", $name, $who, $sizes[$i]); + my $val = $data{$sys}->{$key}->{'MB/s'}; + next if !defined($val); + if($val > $ymax) { + $ymax = $val; + } + $lastx{$who} = $i; + $lasty{$who} = $val; + printf JGR "$i %f (* %s *)\n", $val, $key; + } + my $color = $color{$who}; + printf JGR "marktype none color $color linethickness 2 linetype solid label : $who\n"; + } + my $n = @sizes; + printf JGR "xaxis min -1 max $n size 5 label : text size (bytes)\n"; + printf JGR " no_auto_hash_marks hash_labels fontsize 9\n"; + for($i=0; $i<@sizes; $i+=3) { + printf JGR " hash_at $i hash_label at $i : $sizes[$i]\n"; + } + my $y = 1; + while(10*$y <= $ymax) { + $y = 10*$y; + } + for($i=2; $i<=10; $i++) { + if($i*$y > $ymax) { + $y = $i*$y; + last; + } + } + foreach my $who ("PCRE", "RE2") { + $x1 = $lastx{$who}; + $y1 = $lasty{$who}; + $x1 *= 1.01; + my $v = "vjc"; + if($y1 < 0.05 * $y) { + $v = "vjb"; + $y1 = 0.05 * $y; + } + printf JGR "newstring x $x1 y $y1 hjl $v : $who\n"; + } + printf JGR "yaxis min 0 max $y size 1 label : speed (MB/s)\n"; + printf JGR " hash_labels fontsize 9\n"; + # printf JGR "legend defaults font Times-Roman fontsize 10 x 0 y $y hjl vjt\n"; + + system("jgraph $base.jgr >$base.eps"); # die "system: $!"; + system("gs -dTextAlphaBits=4 -dGraphicsAlphaBits=4 -dEPSCrop -sDEVICE=png16m -r100 -sOutputFile=$base.png -dBATCH -dQUIT -dQUIET -dNOPAUSE $base.eps"); + + printf "\n" + +} + +sub skip() { + while(<>) { + if(/^/) { + print; + last; + } + } +} + +@sys = ("r70", "c2", "wreck", "mini"); +%sysname = ( + "r70" => "AMD Opteron 8214 HE, 2.2 GHz", + "c2" => "Intel Core2 Duo E7200, 2.53 GHz", + "wreck" => "Intel Xeon 5150, 2.66 GHz (Mac Pro)", + "mini" => "Intel Core2 T5600, 1.83 GHz (Mac Mini)", +); + +%func = ( + "table" => \&table, + "graph" => \&graph, + +); + +foreach my $sys (@sys) { + open(F, "benchlog.$sys") || die "open benchlog.$sys: $!"; + my %sysdat; + while() { + if(/^([A-Za-z0-9_\/]+)\s+(\d+)\s+(\d+) ns\/op/) { + my %row; + $row{"name"} = $1; + $row{"iter"} = $2; + $row{"ns/op"} = $3; + if(/([\d.]+) MB\/s/){ + $row{"MB/s"} = $1; + } + $sysdat{$row{"name"}} = \%row; + } + } + close F; + $data{$sys} = \%sysdat; +} + +while(<>) { + print; + if(/^/) { + $func{$1}(); + skip(); + next; + } + if(/^/) { + $func{$1}($2); + skip(); + next; + } +} + diff --git a/extern/re2/doc/README.xkcd b/extern/re2/doc/README.xkcd new file mode 100644 index 0000000000..b50a579a55 --- /dev/null +++ b/extern/re2/doc/README.xkcd @@ -0,0 +1 @@ +xkcd.png is a cropped version of http://xkcd.com/208/ diff --git a/extern/re2/doc/mksyntaxgo b/extern/re2/doc/mksyntaxgo new file mode 100644 index 0000000000..caad9b60b0 --- /dev/null +++ b/extern/re2/doc/mksyntaxgo @@ -0,0 +1,41 @@ +#!/bin/sh + +set -e +out=$GOROOT/src/regexp/syntax/doc.go +cp syntax.txt $out +sam -d $out <<'!' +,x g/NOT SUPPORTED/d +/^Unicode character class/,$d +,s/[«»]//g +,x g/^Possessive repetitions:/d +,x g/\\C/d +,x g/Flag syntax/d +,s/.=(true|false)/flag &/g +,s/^Flags:/ Flag syntax is xyz (set) or -xyz (clear) or xy-z (set xy, clear z). The flags are:\n/ +,s/\n\n\n+/\n\n/g +,x/(^.* .*\n)+/ | awk -F' ' '{printf(" %-14s %s\n", $1, $2)}' +1,2c +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// DO NOT EDIT. This file is generated by mksyntaxgo from the RE2 distribution. + +/* +Package syntax parses regular expressions into parse trees and compiles +parse trees into programs. Most clients of regular expressions will use the +facilities of package regexp (such as Compile and Match) instead of this package. + +Syntax + +The regular expression syntax understood by this package when parsing with the Perl flag is as follows. +Parts of the syntax can be disabled by passing alternate flags to Parse. + +. +$a +*/ +package syntax +. +w +q +! diff --git a/extern/re2/doc/mksyntaxhtml b/extern/re2/doc/mksyntaxhtml new file mode 100644 index 0000000000..0292ea00a8 --- /dev/null +++ b/extern/re2/doc/mksyntaxhtml @@ -0,0 +1,42 @@ +#!/bin/sh + +cp syntax.txt syntax.html +sam -d syntax.html <<'!' +,s/\&/\&/g +,s//\>/g +,s!== (([^()]|\([^()]*\))*)!≡ \1!g +,s!«!!g +,s!»!!g +,s! vim$! VIM!g +,s! pcre$! PCRE!g +,s! perl$! PERL!g +,x g/NOT SUPPORTED/ s!^[^ ]+!&! +,s!NOT SUPPORTED!!g +,s!(^[^ ]+) (.*)\n!\1\2\n!g +,s!.*:$!&!g +,s!^$!!g +,x v// s!.*!&! +1,2c + + + + +RE2 regular expression syntax reference + + +

RE2 regular expression syntax reference

+ + + + + +. +$a +
This page lists the regular expression syntax accepted by RE2.
It also lists syntax accepted by PCRE, PERL, and VIM.
Grayed out expressions are not supported by RE2.
+ + +. +w +q +! diff --git a/extern/re2/doc/mksyntaxwiki b/extern/re2/doc/mksyntaxwiki new file mode 100644 index 0000000000..930b3896e2 --- /dev/null +++ b/extern/re2/doc/mksyntaxwiki @@ -0,0 +1,36 @@ +#!/bin/sh + +cp syntax.txt syntax.wiki +sam -d syntax.wiki <<'!' +,s!`!`````!g +,s!== (([^()]|\([^()]*\))*)!≡ `\1`!g +,s!«!`!g +,s!»!`!g +,s! vim$! VIM!g +,s! pcre$! PCRE!g +,s! perl$! PERL!g +,s!(^[^ ]+) (.*)\n!`\1` \2\n!g +,x g/NOT SUPPORTED/ s!^[^ ]+!&! +,s!NOT SUPPORTED!(&)!g +,s!(^[^ ]+) (.*)\n!\1\2\n!g +,s!.*:$!&!g +,s!^$!!g +,x v// s!.*!&! +1,2c +#summary I define UNIX as “30 definitions of regular expressions living under one roof.” —Don Knuth + + +GENERATED BY mksyntaxwiki. DO NOT EDIT + + + + + + +. +$a +
This page lists the regular expression syntax accepted by RE2.
It also lists syntax accepted by PCRE, PERL, and VIM.
Grayed out expressions are not supported by RE2.
+. +w +q +! diff --git a/extern/re2/doc/syntax.html b/extern/re2/doc/syntax.html new file mode 100644 index 0000000000..aa08b1108b --- /dev/null +++ b/extern/re2/doc/syntax.html @@ -0,0 +1,409 @@ + + + + +RE2 regular expression syntax reference + + +

RE2 regular expression syntax reference

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
This page lists the regular expression syntax accepted by RE2.
It also lists syntax accepted by PCRE, PERL, and VIM.
Grayed out expressions are not supported by RE2.
Single characters:
.any character, possibly including newline (s=true)
[xyz]character class
[^xyz]negated character class
\dPerl character class
\Dnegated Perl character class
[[:alpha:]]ASCII character class
[[:^alpha:]]negated ASCII character class
\pNUnicode character class (one-letter name)
\p{Greek}Unicode character class
\PNnegated Unicode character class (one-letter name)
\P{Greek}negated Unicode character class
Composites:
xyx followed by y
x|yx or y (prefer x)
Repetitions:
x*zero or more x, prefer more
x+one or more x, prefer more
x?zero or one x, prefer one
x{n,m}n or n+1 or ... or m x, prefer more
x{n,}n or more x, prefer more
x{n}exactly n x
x*?zero or more x, prefer fewer
x+?one or more x, prefer fewer
x??zero or one x, prefer zero
x{n,m}?n or n+1 or ... or m x, prefer fewer
x{n,}?n or more x, prefer fewer
x{n}?exactly n x
x{}(≡ x*) VIM
x{-}(≡ x*?) VIM
x{-n}(≡ x{n}?) VIM
x=(≡ x?) VIM
Possessive repetitions:
x*+zero or more x, possessive
x++one or more x, possessive
x?+zero or one x, possessive
x{n,m}+n or ... or m x, possessive
x{n,}+n or more x, possessive
x{n}+exactly n x, possessive
Grouping:
(re)numbered capturing group
(?P<name>re)named & numbered capturing group
(?<name>re)named & numbered capturing group
(?'name're)named & numbered capturing group
(?:re)non-capturing group
(?flags)set flags within current group; non-capturing
(?flags:re)set flags during re; non-capturing
(?#text)comment
(?|x|y|z)branch numbering reset
(?>re)possessive match of re
re@>possessive match of re VIM
%(re)non-capturing group VIM
Flags:
icase-insensitive (default false)
mmulti-line mode: ^ and $ match begin/end line in addition to begin/end text (default false)
slet . match \n (default false)
Uungreedy: swap meaning of x* and x*?, x+ and x+?, etc (default false)
Flag syntax is xyz (set) or -xyz (clear) or xy-z (set xy, clear z).
Empty strings:
^at beginning of text or line (m=true)
$at end of text (like \z not \Z) or line (m=true)
\Aat beginning of text
\bat word boundary (\w on one side and \W, \A, or \z on the other)
\Bnot a word boundary
\Gat beginning of subtext being searched PCRE
\Gat end of last match PERL
\Zat end of text, or before newline at end of text
\zat end of text
(?=re)before text matching re
(?!re)before text not matching re
(?<=re)after text matching re
(?<!re)after text not matching re
re&before text matching re VIM
re@=before text matching re VIM
re@!before text not matching re VIM
re@<=after text matching re VIM
re@<!after text not matching re VIM
\zssets start of match (= \K) VIM
\zesets end of match VIM
\%^beginning of file VIM
\%$end of file VIM
\%Von screen VIM
\%#cursor position VIM
\%'mmark m position VIM
\%23lin line 23 VIM
\%23cin column 23 VIM
\%23vin virtual column 23 VIM
Escape sequences:
\abell (≡ \007)
\fform feed (≡ \014)
\thorizontal tab (≡ \011)
\nnewline (≡ \012)
\rcarriage return (≡ \015)
\vvertical tab character (≡ \013)
\*literal *, for any punctuation character *
\123octal character code (up to three digits)
\x7Fhex character code (exactly two digits)
\x{10FFFF}hex character code
\Cmatch a single byte even in UTF-8 mode
\Q...\Eliteral text ... even if ... has punctuation
\1backreference
\bbackspace (use \010)
\cKcontrol char ^K (use \001 etc)
\eescape (use \033)
\g1backreference
\g{1}backreference
\g{+1}backreference
\g{-1}backreference
\g{name}named backreference
\g<name>subroutine call
\g'name'subroutine call
\k<name>named backreference
\k'name'named backreference
\lXlowercase X
\uxuppercase x
\L...\Elowercase text ...
\Kreset beginning of $0
\N{name}named Unicode character
\Rline break
\U...\Eupper case text ...
\Xextended Unicode sequence
\%d123decimal character 123 VIM
\%xFFhex character FF VIM
\%o123octal character 123 VIM
\%u1234Unicode character 0x1234 VIM
\%U12345678Unicode character 0x12345678 VIM
Character class elements:
xsingle character
A-Zcharacter range (inclusive)
\dPerl character class
[:foo:]ASCII character class foo
\p{Foo}Unicode character class Foo
\pFUnicode character class F (one-letter name)
Named character classes as character class elements:
[\d]digits (≡ \d)
[^\d]not digits (≡ \D)
[\D]not digits (≡ \D)
[^\D]not not digits (≡ \d)
[[:name:]]named ASCII class inside character class (≡ [:name:])
[^[:name:]]named ASCII class inside negated character class (≡ [:^name:])
[\p{Name}]named Unicode property inside character class (≡ \p{Name})
[^\p{Name}]named Unicode property inside negated character class (≡ \P{Name})
Perl character classes:
\ddigits (≡ [0-9])
\Dnot digits (≡ [^0-9])
\swhitespace (≡ [\t\n\f\r ])
\Snot whitespace (≡ [^\t\n\f\r ])
\wword characters (≡ [0-9A-Za-z_])
\Wnot word characters (≡ [^0-9A-Za-z_])
\hhorizontal space
\Hnot horizontal space
\vvertical space
\Vnot vertical space
ASCII character classes:
[[:alnum:]]alphanumeric (≡ [0-9A-Za-z])
[[:alpha:]]alphabetic (≡ [A-Za-z])
[[:ascii:]]ASCII (≡ [\x00-\x7F])
[[:blank:]]blank (≡ [\t ])
[[:cntrl:]]control (≡ [\x00-\x1F\x7F])
[[:digit:]]digits (≡ [0-9])
[[:graph:]]graphical (≡ [!-~] == [A-Za-z0-9!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~])
[[:lower:]]lower case (≡ [a-z])
[[:print:]]printable (≡ [ -~] == [ [:graph:]])
[[:punct:]]punctuation (≡ [!-/:-@[-`{-~])
[[:space:]]whitespace (≡ [\t\n\v\f\r ])
[[:upper:]]upper case (≡ [A-Z])
[[:word:]]word characters (≡ [0-9A-Za-z_])
[[:xdigit:]]hex digit (≡ [0-9A-Fa-f])
Unicode character class names--general category:
Cother
Cccontrol
Cfformat
Cnunassigned code points
Coprivate use
Cssurrogate
Lletter
LCcased letter
L&cased letter
Lllowercase letter
Lmmodifier letter
Loother letter
Lttitlecase letter
Luuppercase letter
Mmark
Mcspacing mark
Meenclosing mark
Mnnon-spacing mark
Nnumber
Nddecimal number
Nlletter number
Noother number
Ppunctuation
Pcconnector punctuation
Pddash punctuation
Peclose punctuation
Pffinal punctuation
Piinitial punctuation
Poother punctuation
Psopen punctuation
Ssymbol
Sccurrency symbol
Skmodifier symbol
Smmath symbol
Soother symbol
Zseparator
Zlline separator
Zpparagraph separator
Zsspace separator
Unicode character class names--scripts:
ArabicArabic
ArmenianArmenian
BalineseBalinese
BamumBamum
BatakBatak
BengaliBengali
BopomofoBopomofo
BrahmiBrahmi
BrailleBraille
BugineseBuginese
BuhidBuhid
Canadian_AboriginalCanadian Aboriginal
CarianCarian
ChakmaChakma
ChamCham
CherokeeCherokee
Commoncharacters not specific to one script
CopticCoptic
CuneiformCuneiform
CypriotCypriot
CyrillicCyrillic
DeseretDeseret
DevanagariDevanagari
Egyptian_HieroglyphsEgyptian Hieroglyphs
EthiopicEthiopic
GeorgianGeorgian
GlagoliticGlagolitic
GothicGothic
GreekGreek
GujaratiGujarati
GurmukhiGurmukhi
HanHan
HangulHangul
HanunooHanunoo
HebrewHebrew
HiraganaHiragana
Imperial_AramaicImperial Aramaic
Inheritedinherit script from previous character
Inscriptional_PahlaviInscriptional Pahlavi
Inscriptional_ParthianInscriptional Parthian
JavaneseJavanese
KaithiKaithi
KannadaKannada
KatakanaKatakana
Kayah_LiKayah Li
KharoshthiKharoshthi
KhmerKhmer
LaoLao
LatinLatin
LepchaLepcha
LimbuLimbu
Linear_BLinear B
LycianLycian
LydianLydian
MalayalamMalayalam
MandaicMandaic
Meetei_MayekMeetei Mayek
Meroitic_CursiveMeroitic Cursive
Meroitic_HieroglyphsMeroitic Hieroglyphs
MiaoMiao
MongolianMongolian
MyanmarMyanmar
New_Tai_LueNew Tai Lue (aka Simplified Tai Lue)
NkoNko
OghamOgham
Ol_ChikiOl Chiki
Old_ItalicOld Italic
Old_PersianOld Persian
Old_South_ArabianOld South Arabian
Old_TurkicOld Turkic
OriyaOriya
OsmanyaOsmanya
Phags_Pa'Phags Pa
PhoenicianPhoenician
RejangRejang
RunicRunic
SaurashtraSaurashtra
SharadaSharada
ShavianShavian
SinhalaSinhala
Sora_SompengSora Sompeng
SundaneseSundanese
Syloti_NagriSyloti Nagri
SyriacSyriac
TagalogTagalog
TagbanwaTagbanwa
Tai_LeTai Le
Tai_ThamTai Tham
Tai_VietTai Viet
TakriTakri
TamilTamil
TeluguTelugu
ThaanaThaana
ThaiThai
TibetanTibetan
TifinaghTifinagh
UgariticUgaritic
VaiVai
YiYi
Vim character classes:
\iidentifier character VIM
\I\i except digits VIM
\kkeyword character VIM
\K\k except digits VIM
\ffile name character VIM
\F\f except digits VIM
\pprintable character VIM
\P\p except digits VIM
\swhitespace character (≡ [ \t]) VIM
\Snon-white space character (≡ [^ \t]) VIM
\ddigits (≡ [0-9]) VIM
\Dnot \d VIM
\xhex digits (≡ [0-9A-Fa-f]) VIM
\Xnot \x VIM
\ooctal digits (≡ [0-7]) VIM
\Onot \o VIM
\wword character VIM
\Wnot \w VIM
\hhead of word character VIM
\Hnot \h VIM
\aalphabetic VIM
\Anot \a VIM
\llowercase VIM
\Lnot lowercase VIM
\uuppercase VIM
\Unot uppercase VIM
\_x\x plus newline, for any x VIM
Vim flags:
\cignore case VIM
\Cmatch case VIM
\mmagic VIM
\Mnomagic VIM
\vverymagic VIM
\Vverynomagic VIM
\Zignore differences in Unicode combining characters VIM
Magic:
(?{code})arbitrary Perl code PERL
(??{code})postponed arbitrary Perl code PERL
(?n)recursive call to regexp capturing group n
(?+n)recursive call to relative group +n
(?-n)recursive call to relative group -n
(?C)PCRE callout PCRE
(?R)recursive call to entire regexp (≡ (?0))
(?&name)recursive call to named group
(?P=name)named backreference
(?P>name)recursive call to named group
(?(cond)true|false)conditional branch
(?(cond)true)conditional branch
(*ACCEPT)make regexps more like Prolog
(*COMMIT)
(*F)
(*FAIL)
(*MARK)
(*PRUNE)
(*SKIP)
(*THEN)
(*ANY)set newline convention
(*ANYCRLF)
(*CR)
(*CRLF)
(*LF)
(*BSR_ANYCRLF)set \R convention PCRE
(*BSR_UNICODE) PCRE
+ + diff --git a/extern/re2/doc/syntax.txt b/extern/re2/doc/syntax.txt new file mode 100644 index 0000000000..cb04bbf05e --- /dev/null +++ b/extern/re2/doc/syntax.txt @@ -0,0 +1,452 @@ +RE2 regular expression syntax reference +-------------------------­-------­----- + +Single characters: +. any character, possibly including newline (s=true) +[xyz] character class +[^xyz] negated character class +\d Perl character class +\D negated Perl character class +[[:alpha:]] ASCII character class +[[:^alpha:]] negated ASCII character class +\pN Unicode character class (one-letter name) +\p{Greek} Unicode character class +\PN negated Unicode character class (one-letter name) +\P{Greek} negated Unicode character class + +Composites: +xy «x» followed by «y» +x|y «x» or «y» (prefer «x») + +Repetitions: +x* zero or more «x», prefer more +x+ one or more «x», prefer more +x? zero or one «x», prefer one +x{n,m} «n» or «n»+1 or ... or «m» «x», prefer more +x{n,} «n» or more «x», prefer more +x{n} exactly «n» «x» +x*? zero or more «x», prefer fewer +x+? one or more «x», prefer fewer +x?? zero or one «x», prefer zero +x{n,m}? «n» or «n»+1 or ... or «m» «x», prefer fewer +x{n,}? «n» or more «x», prefer fewer +x{n}? exactly «n» «x» +x{} (== x*) NOT SUPPORTED vim +x{-} (== x*?) NOT SUPPORTED vim +x{-n} (== x{n}?) NOT SUPPORTED vim +x= (== x?) NOT SUPPORTED vim + +Implementation restriction: The counting forms «x{n,m}», «x{n,}», and «x{n}» +reject forms that create a minimum or maximum repetition count above 1000. +Unlimited repetitions are not subject to this restriction. + +Possessive repetitions: +x*+ zero or more «x», possessive NOT SUPPORTED +x++ one or more «x», possessive NOT SUPPORTED +x?+ zero or one «x», possessive NOT SUPPORTED +x{n,m}+ «n» or ... or «m» «x», possessive NOT SUPPORTED +x{n,}+ «n» or more «x», possessive NOT SUPPORTED +x{n}+ exactly «n» «x», possessive NOT SUPPORTED + +Grouping: +(re) numbered capturing group (submatch) +(?Pre) named & numbered capturing group (submatch) +(?re) named & numbered capturing group (submatch) NOT SUPPORTED +(?'name're) named & numbered capturing group (submatch) NOT SUPPORTED +(?:re) non-capturing group +(?flags) set flags within current group; non-capturing +(?flags:re) set flags during re; non-capturing +(?#text) comment NOT SUPPORTED +(?|x|y|z) branch numbering reset NOT SUPPORTED +(?>re) possessive match of «re» NOT SUPPORTED +re@> possessive match of «re» NOT SUPPORTED vim +%(re) non-capturing group NOT SUPPORTED vim + +Flags: +i case-insensitive (default false) +m multi-line mode: «^» and «$» match begin/end line in addition to begin/end text (default false) +s let «.» match «\n» (default false) +U ungreedy: swap meaning of «x*» and «x*?», «x+» and «x+?», etc (default false) +Flag syntax is «xyz» (set) or «-xyz» (clear) or «xy-z» (set «xy», clear «z»). + +Empty strings: +^ at beginning of text or line («m»=true) +$ at end of text (like «\z» not «\Z») or line («m»=true) +\A at beginning of text +\b at ASCII word boundary («\w» on one side and «\W», «\A», or «\z» on the other) +\B not at ASCII word boundary +\G at beginning of subtext being searched NOT SUPPORTED pcre +\G at end of last match NOT SUPPORTED perl +\Z at end of text, or before newline at end of text NOT SUPPORTED +\z at end of text +(?=re) before text matching «re» NOT SUPPORTED +(?!re) before text not matching «re» NOT SUPPORTED +(?<=re) after text matching «re» NOT SUPPORTED +(? subroutine call NOT SUPPORTED +\g'name' subroutine call NOT SUPPORTED +\k named backreference NOT SUPPORTED +\k'name' named backreference NOT SUPPORTED +\lX lowercase «X» NOT SUPPORTED +\ux uppercase «x» NOT SUPPORTED +\L...\E lowercase text «...» NOT SUPPORTED +\K reset beginning of «$0» NOT SUPPORTED +\N{name} named Unicode character NOT SUPPORTED +\R line break NOT SUPPORTED +\U...\E upper case text «...» NOT SUPPORTED +\X extended Unicode sequence NOT SUPPORTED + +\%d123 decimal character 123 NOT SUPPORTED vim +\%xFF hex character FF NOT SUPPORTED vim +\%o123 octal character 123 NOT SUPPORTED vim +\%u1234 Unicode character 0x1234 NOT SUPPORTED vim +\%U12345678 Unicode character 0x12345678 NOT SUPPORTED vim + +Character class elements: +x single character +A-Z character range (inclusive) +\d Perl character class +[:foo:] ASCII character class «foo» +\p{Foo} Unicode character class «Foo» +\pF Unicode character class «F» (one-letter name) + +Named character classes as character class elements: +[\d] digits (== \d) +[^\d] not digits (== \D) +[\D] not digits (== \D) +[^\D] not not digits (== \d) +[[:name:]] named ASCII class inside character class (== [:name:]) +[^[:name:]] named ASCII class inside negated character class (== [:^name:]) +[\p{Name}] named Unicode property inside character class (== \p{Name}) +[^\p{Name}] named Unicode property inside negated character class (== \P{Name}) + +Perl character classes (all ASCII-only): +\d digits (== [0-9]) +\D not digits (== [^0-9]) +\s whitespace (== [\t\n\f\r ]) +\S not whitespace (== [^\t\n\f\r ]) +\w word characters (== [0-9A-Za-z_]) +\W not word characters (== [^0-9A-Za-z_]) + +\h horizontal space NOT SUPPORTED +\H not horizontal space NOT SUPPORTED +\v vertical space NOT SUPPORTED +\V not vertical space NOT SUPPORTED + +ASCII character classes: +[[:alnum:]] alphanumeric (== [0-9A-Za-z]) +[[:alpha:]] alphabetic (== [A-Za-z]) +[[:ascii:]] ASCII (== [\x00-\x7F]) +[[:blank:]] blank (== [\t ]) +[[:cntrl:]] control (== [\x00-\x1F\x7F]) +[[:digit:]] digits (== [0-9]) +[[:graph:]] graphical (== [!-~] == [A-Za-z0-9!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]) +[[:lower:]] lower case (== [a-z]) +[[:print:]] printable (== [ -~] == [ [:graph:]]) +[[:punct:]] punctuation (== [!-/:-@[-`{-~]) +[[:space:]] whitespace (== [\t\n\v\f\r ]) +[[:upper:]] upper case (== [A-Z]) +[[:word:]] word characters (== [0-9A-Za-z_]) +[[:xdigit:]] hex digit (== [0-9A-Fa-f]) + +Unicode character class names--general category: +C other +Cc control +Cf format +Cn unassigned code points NOT SUPPORTED +Co private use +Cs surrogate +L letter +LC cased letter NOT SUPPORTED +L& cased letter NOT SUPPORTED +Ll lowercase letter +Lm modifier letter +Lo other letter +Lt titlecase letter +Lu uppercase letter +M mark +Mc spacing mark +Me enclosing mark +Mn non-spacing mark +N number +Nd decimal number +Nl letter number +No other number +P punctuation +Pc connector punctuation +Pd dash punctuation +Pe close punctuation +Pf final punctuation +Pi initial punctuation +Po other punctuation +Ps open punctuation +S symbol +Sc currency symbol +Sk modifier symbol +Sm math symbol +So other symbol +Z separator +Zl line separator +Zp paragraph separator +Zs space separator + +Unicode character class names--scripts: +Adlam +Ahom +Anatolian_Hieroglyphs +Arabic +Armenian +Avestan +Balinese +Bamum +Bassa_Vah +Batak +Bengali +Bhaiksuki +Bopomofo +Brahmi +Braille +Buginese +Buhid +Canadian_Aboriginal +Carian +Caucasian_Albanian +Chakma +Cham +Cherokee +Common +Coptic +Cuneiform +Cypriot +Cyrillic +Deseret +Devanagari +Dogra +Duployan +Egyptian_Hieroglyphs +Elbasan +Elymaic +Ethiopic +Georgian +Glagolitic +Gothic +Grantha +Greek +Gujarati +Gunjala_Gondi +Gurmukhi +Han +Hangul +Hanifi_Rohingya +Hanunoo +Hatran +Hebrew +Hiragana +Imperial_Aramaic +Inherited +Inscriptional_Pahlavi +Inscriptional_Parthian +Javanese +Kaithi +Kannada +Katakana +Kayah_Li +Kharoshthi +Khmer +Khojki +Khudawadi +Lao +Latin +Lepcha +Limbu +Linear_A +Linear_B +Lisu +Lycian +Lydian +Mahajani +Makasar +Malayalam +Mandaic +Manichaean +Marchen +Masaram_Gondi +Medefaidrin +Meetei_Mayek +Mende_Kikakui +Meroitic_Cursive +Meroitic_Hieroglyphs +Miao +Modi +Mongolian +Mro +Multani +Myanmar +Nabataean +Nandinagari +New_Tai_Lue +Newa +Nko +Nushu +Nyiakeng_Puachue_Hmong +Ogham +Ol_Chiki +Old_Hungarian +Old_Italic +Old_North_Arabian +Old_Permic +Old_Persian +Old_Sogdian +Old_South_Arabian +Old_Turkic +Oriya +Osage +Osmanya +Pahawh_Hmong +Palmyrene +Pau_Cin_Hau +Phags_Pa +Phoenician +Psalter_Pahlavi +Rejang +Runic +Samaritan +Saurashtra +Sharada +Shavian +Siddham +SignWriting +Sinhala +Sogdian +Sora_Sompeng +Soyombo +Sundanese +Syloti_Nagri +Syriac +Tagalog +Tagbanwa +Tai_Le +Tai_Tham +Tai_Viet +Takri +Tamil +Tangut +Telugu +Thaana +Thai +Tibetan +Tifinagh +Tirhuta +Ugaritic +Vai +Wancho +Warang_Citi +Yi +Zanabazar_Square + +Vim character classes: +\i identifier character NOT SUPPORTED vim +\I «\i» except digits NOT SUPPORTED vim +\k keyword character NOT SUPPORTED vim +\K «\k» except digits NOT SUPPORTED vim +\f file name character NOT SUPPORTED vim +\F «\f» except digits NOT SUPPORTED vim +\p printable character NOT SUPPORTED vim +\P «\p» except digits NOT SUPPORTED vim +\s whitespace character (== [ \t]) NOT SUPPORTED vim +\S non-white space character (== [^ \t]) NOT SUPPORTED vim +\d digits (== [0-9]) vim +\D not «\d» vim +\x hex digits (== [0-9A-Fa-f]) NOT SUPPORTED vim +\X not «\x» NOT SUPPORTED vim +\o octal digits (== [0-7]) NOT SUPPORTED vim +\O not «\o» NOT SUPPORTED vim +\w word character vim +\W not «\w» vim +\h head of word character NOT SUPPORTED vim +\H not «\h» NOT SUPPORTED vim +\a alphabetic NOT SUPPORTED vim +\A not «\a» NOT SUPPORTED vim +\l lowercase NOT SUPPORTED vim +\L not lowercase NOT SUPPORTED vim +\u uppercase NOT SUPPORTED vim +\U not uppercase NOT SUPPORTED vim +\_x «\x» plus newline, for any «x» NOT SUPPORTED vim + +Vim flags: +\c ignore case NOT SUPPORTED vim +\C match case NOT SUPPORTED vim +\m magic NOT SUPPORTED vim +\M nomagic NOT SUPPORTED vim +\v verymagic NOT SUPPORTED vim +\V verynomagic NOT SUPPORTED vim +\Z ignore differences in Unicode combining characters NOT SUPPORTED vim + +Magic: +(?{code}) arbitrary Perl code NOT SUPPORTED perl +(??{code}) postponed arbitrary Perl code NOT SUPPORTED perl +(?n) recursive call to regexp capturing group «n» NOT SUPPORTED +(?+n) recursive call to relative group «+n» NOT SUPPORTED +(?-n) recursive call to relative group «-n» NOT SUPPORTED +(?C) PCRE callout NOT SUPPORTED pcre +(?R) recursive call to entire regexp (== (?0)) NOT SUPPORTED +(?&name) recursive call to named group NOT SUPPORTED +(?P=name) named backreference NOT SUPPORTED +(?P>name) recursive call to named group NOT SUPPORTED +(?(cond)true|false) conditional branch NOT SUPPORTED +(?(cond)true) conditional branch NOT SUPPORTED +(*ACCEPT) make regexps more like Prolog NOT SUPPORTED +(*COMMIT) NOT SUPPORTED +(*F) NOT SUPPORTED +(*FAIL) NOT SUPPORTED +(*MARK) NOT SUPPORTED +(*PRUNE) NOT SUPPORTED +(*SKIP) NOT SUPPORTED +(*THEN) NOT SUPPORTED +(*ANY) set newline convention NOT SUPPORTED +(*ANYCRLF) NOT SUPPORTED +(*CR) NOT SUPPORTED +(*CRLF) NOT SUPPORTED +(*LF) NOT SUPPORTED +(*BSR_ANYCRLF) set \R convention NOT SUPPORTED pcre +(*BSR_UNICODE) NOT SUPPORTED pcre + diff --git a/extern/re2/doc/xkcd.png b/extern/re2/doc/xkcd.png new file mode 100644 index 0000000000000000000000000000000000000000..6249e8e0d325415ec447d16c0cb201515b834b63 GIT binary patch literal 26496 zcmY(KRajfW+NcxUU4py26)0NV-QC@a6)5iR?h+h|ySo-BF2$Wf(PBkTy7&Gs&PDPh zlUcLYd^$7v-Y69%X;dUaBme+_Dk~$Q1^_^rLq1O-KtbMXeV_vY0N{$PxVVa}xH!3r ztFx7@gCzjq1q@48q=zvDjlDNvVNy&fI-?0A)wVQGn68;wE>J%yeV>VMX(2QB^gZmC z{p_YXD9b+%3(o)#FCC5J`bB$kWXx;Jd!u#nTltoEz-__%tsvK|fM~w$VqrtM#2m;1gs3!O9NxK;P0&`mJQ z+I9>Vh6fJIRKz0OyPzSWxfpSKqzJ0#F6q)J)L`=x42k>i3GK~C?wHR|3}ocuFCxtL zPu;h@H$a>0 zth>SF>uFf2{!ZW5>%8KRf3{`RKj1ah8?x@~Ac2gagPSkstLoq;wE=6=o#^ptU3XY+ z(CF5pR-e``PHn9HY4A`9qAP!#OYmU^qJ)zhZeA5aU$`Gf1cQnZ2FN*!29(Bi84R!A zHy6`Sh=BoW#vh84bW=PQ$SND&Lf7Yqf3jxqenI_w|5N=h+~V48-*4A%uiu|x&EOKc zcp8tkA(}>Xk}jq~Qg;*o1F)m-L2rso{X(@sx~H#S&Am zSAmg~1S8K&zH0GDHUGG4`#iVuGArwS!|I6R{UP9;S6Ns#2rQ~b&X9;BgVbEqZ@w{X?f$>7 z-0C|IeebUY?=QzkUkC)^ab&>6V|Cena!Ay$&k8{jlfu|u?a|6$;%}EpwX%@L@B*mY z8G%b^bV2r)i)p+WJXq2c&&f4P%k&h;=n4q1%sKgE!XIej5TAB#xycVh(1rOJdS?A; zVUwX8>8~EgMBX?OycPZ&7QEfjU8%%PxTu*JFkFSw2^Oulz<~{124Km2PKp*;G$vAa z!Tq5s0K7q&sbN)Vl9oxv>Qp0rjAaA2??les90QGW)X@9Vz@fiU%D%-6E(?mhnqMCt z&k_)}eQ&=j``@JC>zD*_>$YLhnI}W-D+29@^S(e_!?A7i>t)YU{$9Rw`)+vGW$)MB zFnp!@8a3LXeF@mlyF5ElIfisY&oyM$_QG`TfPXVc5e-W4J)jb}XjW$; ze!F~syPTJ2^LskXP=>;6Ou&{8VrgC73&mEd@aS8kG;C5{yg8VKh~DI!Au*Q2Dhalnm6$MrLKEhh;K`9a3N z)!~j>WhbTJS{)e!oZ+f9ho zJa%orIw?KpNIOm=t~Jf?{u|20Kf_@uRBZ?H({G3o`d*LV|9Z#Hzk<=0bZS}BaubVL ze4hFLdsD0J=x;wS3v6A-=Q#KBe|-Ou;P0N}*qS9FhSEJ7@OE3Uss8#U;NeSxp93Kn zPU%&(f<0n{-nZ0Yzg>L?J>W4UXW97mOdCHF?5CJXylP!;r<6$yYO$^^{^vCIbv^eH zA}_mZjP>IDQK=-6DRs$EA%*>@yg#qXczaGEw&2pFzxdEAo&RYu&qGiQD3djRfDi}E zby(9hqi0{+G9&cxlg;l`9@@s6@(>W2t4R%;&2ZHZjeZU>O?kGSyOtHBu4&FGiQ=Az ztsK#{Tnh6Ps1!w@`qABja=D+bbI+gYZ0+tn+%aOeX@E?!=Ohq=zgh-~TP5(f%@8VY z_uahw`%}TYcLa$i-(DmY_1<99e!Rd3@Jz)&%gp!o-1ht8Mor&vQXFICIzglCccu<6 znwiP~FM3cz$#6xF^b_JPUB5ejK_N2(Xd#mSqE43Qb5_~5<-?R`KX_0=pC*zF6;KLj zyRytMZ0$ejd;8Vrzl&@PLQ$|2n@kOOMSBWRD}2X!JXH?xnU|&WJ2xlmI?n%^u49H6 zT_W;!oiN4L_u?GzcVj<6#Q*j??_~gXV>q}!QZ-OBnT)SGl&ZS;^!A~BsePO-i}CcN{#zue0IDaP*^zayYlsN%Mq{IFeKZqwB$m=NHr)_9v2@4v~OL1TPgs`HoBe48?UQ+tGE^BP!y zgwLm=9OWFEE7;4%#TBUK_ov$Ubq8K#yekdH`zXuW|Mi#g z`&@+Z@Q2e#X)viMw4Elk+A;!XNdd>4wT6ThDKgU4TEjN-Em2O7mXWt5?F4?`;|QMC z&ui`88{DbGW0Ci#B`tkN=1zfjC3f8)N~=wQ-|f)u6#KLO_gn}PM&(lt$-xylPMuM` zzX@i&;D~)r<&BYY3$v-bC<|fGub^fA#qmSe^`W-DS4UD!Iekce0In`+8((x&gJ`K} zIu^cE3pIP1MN%YcYwN06-Pg@uvGa67r?PJCshh4-ED^-6lV$Ft!6SbCEmpW(-oq)DG3VQ@IR<+LnkzD12dH}7H9932Zh^7-}83hym^ z-KT8Tv2b#(HM0W0dtmftTbL~95Qt&|O|&x%tv6{bxfF;YighW9e2uBqbEa9BOi~=E z$&gl-o5+b6gC+RTF$*g)31iESIPNXy8Hup;t$i>JIQeOUvTW{ylCB!P|_`g0sKv? z^DU)_b|0s_tl`v zfH{q%X`+-85cWv|SCmEa$UFJ z4vST#(o~A2FpMj(${Vm-C77iA&KjcAEpz+*nD;m8T7`m0<$4^e=4Obk+7xs6 zA|PA#Ks*E)W+Ni@z)6&45IzIeEKLs~$JAb_?}WV3ASzJ6AvcS{D2r>ohK+H=X<@qG ziXlokUQOSoD9oaP(x%>Ze?OM%98r}AIg!TX&yx0~ zfl03;7!Z30x@dNxE*j;=w5=eD9ug!J;P%%t8W=bEks6LRV5(egkrGUO%<;Vz3+`!@ z$k}%%2wrbn_$mlpp}JxwN;epi_iiAeunB72_i6xspN&PpCHs;{@qAvYS)+t+H(I}8#YQx_6 zub%iad8--cqPPLW5II{i%fL;oj;#fZpxj(Inc$5J_Z&bDSMn3j{%Fv;8F>M$o?L9E z%u8W~~V6R?9e2(LXtADbndVQo^+oN6(x&u#J)Sct2dTOd6M@a61Pt4j{a^lXh9N*L7h(5?OA+6FF4&ARvXLxblvAx!4;d zV4~&&F(?YYd}>~x$?xw)oAeNa{0vMj4!gs-IIi-$R+uc=$JI|{n{6K+rr<=l6usZ~$ebhwlVU#vQKWOrK?SDDxPt3-dLc z?&0^6;6vtf=@H=@=;lYV=FC}z>(-wwWegl=NGDaT(qLQ)FpUkE!{4qcH6ivTLLWAo zdO;64;)kqUcpa1!du~Wsp$5@QD0f3|{d4HO4u7BuAymap8sc%PtpH3*f@q>rfAygC zqi%j=hI~^|wpZ;R_3oBB3${~OsM;cCumoNcvfrH;w8anh3w*p1DUsJOg=|999f6_8 z0ksoB%W6i3IR}e`A~c$-&sdeqORP2Y!qosZ&$$>CY8gFFbffZv=Uw!GNU}CnEMU-~ zQ`ZHY+|&k&LIxBy3e^HkjqJfdKHFx31%euv;5EzyPIvW*}(O zWq|2iXlCShcV=KejTIxB#=*H#Hoc^y!J7m}t!y~mFtDxoOgaycM}}&hg(;vYFzv!2 z0$VOS&~PPcb)aR<=#?Uqr})G=!D$~jqbpt*r?*ynzgH0tCbM;(P{t5ScC-coxyD+o zVWfY#ifU&#w(@yqxoRb6`OlSVrOFH%OF)5wMzv(|QV4i=Dskr@n>+?HKtOh@mU+@~ zxpbaA78gFh_ELguiAf3lLdw8na-<=>EqD})#CgGo^a|A5M8h21QqgO<+lTG9i*062 zBxUOYRSbKp)B=shX8-NAmt;)$BW6a&{zQEW z=ojyC$H-dXZ&dV{>L~d6}0EEVhHK{w@<$bWVi^(H1AG5pFGwO8yPYn zJdXe9gArgWliHzW7c^#)#w#}rLYE5LwdPh!ybAW5cP5CV(Khd=9?T`UU&IAT*aqKn z%~17ts_E_7Xac9L5X|EoF>H(a33A4(>@&*T1V3-Pi+@Z_#oDMEkc1yHo_K&Q?q%TV zI&2H5*zU;7+tSmuOIU2b)H?vZH=n^0-?SEUbLEY;H|_u6fpT)F3YpAQ$L2dQ-;O6< z^CMf)F-I-&`&HZKG%a@d+A-@l^GtLbVe`1$)|)0VuLldmQhl+p{pKkgN+lG%#~wQC zd&H&LNX~Q2=^1Kg(KkfxKRhXr@*R5 zpyK#VF{?ehU%RAY$PGuI>InKigL~DUwGQ|airwwu%@YyCr`@<5=xx{cdS^-t!+@IM zv8*c{KtZE?avC3^HIM2RtBR}|-Vqk!Of=EIQpduvY7}YH!*m9duh>e@tz;gkI1^bp zld^>*@Mtvc6E_r+dNl;!xafIO;!&W6Q!$6?DRYEtp+>mR-Hd$&qmNzYlZ;Isg(X_p zwlk(YwuYR-kW~yDSxY&@tTr#r8t|PH3B~#;9i&2$VJ^QUc0G=ET)rG0OKb2dI}Z5G_jNKz~>=;_R4A-}et2CstI* zYAnVehZL<-TRrb#y~&$}wtB;=)wGtx3dKk^M4N3M_CAe7P;I#kVS~`aKt>AgF{EAGi_gLt%=+aS9 zYbs5oToO3xdHNLKF@7kYRGnFXY`wtx`Q&)Q&wXjJYWr)u!EH-837?^U?8TzFm0gQik0arX?P z&WTNaikY(8ws20+w4Nk#>6Ekosim!;kJAk^=8n}Uv1rCT2%Nz^+}NTQb)?@U=nfu(Q8>fyYl2vvWFKGx(&zY!mqUo>&Kh?v3L9(LVB@;tTe)>9k8~XFQ#sty`+T zN&^0*w3dA&!yP|%U8++CrB%j(MtShyKq`(;X2jF$6b{SeVy9Z2{q`7BPyzDglpc0gAmtSc{ z3?pCzH(I^37jllMD<8`~ghm!;dAscLK1?UPTWc%TTdA9gVzwpwY&O{ZVIykDh0-V5 zd54WE?ra(<0g5Uno?Fkd3U;zo6B8TtTdyPPuf|P{P6zGcdn#>v!rKL5V$>mg?d==k zKNl`#!)VvZ6!@ea*>LwcSxcFEAVALaBiBoi^B2fY@yi^Wu0duyGth=6<}m7e<6WE9 zjm$eS=>-$su$uhiiGngG*=0W3wvwlF3@3(|26F6o^kHVFX<^nSD4vkO3D{faLs~I# zxyZ0y67w29O<_NdbH(jebW}(H7Fp})xS0tP@PO!I?+HV;egKZ>(N>eeXMNs_L9^kk zs#?m^ZTbAej2{kD)b4R$xz`~ZH2=;bDB29_a}>Vl$K>L`7O}XZtzp!D2Nexwk{Djf@##OC7dd zIr#8uxAqI(*yyae!cHHUW1SkiLii3B&WDG}Yd=&=5!=*1e`I9Jm5ewYL^aK6$O05N zs^!o|N)jF|0PR@Qj&WX;YkTpwe%>8y*(InnXx1tXq(!^2e5cQ#@Zz1-aNaFwTX3 zHA+LPjgl6!@=8o*Q|6VK=nI$71Qe3Ia@254XX+t6%Esy!$6>g9tASzlLS&&3&>(}k^=)gTqoK}6^cTM89x?{0sv~1S&Vppd zWVDn#u7T5y^)VTP4IPQeI+ebpqBNvsc5p4_pdozZI=l3}Ddq8#1%OyTzF~POTEtzzq533GDSt;p;wLFuwYRu+lKv8EvmsuDX{3VX zX$nychDbk(0)n43G@e)+2nAqxtKmn`SAI5EcQGN>aGrL3##gn zOwki9OWLGP3p!JKl>Ye4Ih8dkTyHX`Wn(f9TJLIV610a^yA*?-f$YrNFytSM5Z_HR zDn#GqJ5$SUydDf{!7)o`=YVBsxF5(+aY=)Fz!0UZfu`7^rUUkm&xe!B5v4Ihk}mtS zbsTIb^dA|)@Uc`GTP1>MQcJ=oE0xp=o>dtV@)0uDT= z2(C-%Mb6=AZpS=z2GI_*dJIjPdq7R>DRO(j!or9$^k_^hQ52TXFGOUigc)0>G%oLo z_!deTEkH>|r(tH8blA|PyH|WsGNMs?3p;l}97PJT5zwWZ*A4J9pSQlS*Jb+7R2N-B zmGMCY6}Dt;uBH^dGSZc84;3GQx(B|GVV>Ol_f3k^W7L-O z5?cY9G*CTI1+$NuYuEc3R(!|#$3?0c>>fgFFzfZ?G{^4eJVIF%RMt-`IFVt)LE*%R z)1jIgixSUa9yEXD5KCS1juI-x-r`fk(pVk)ed&un4q4)h(-7GP6S_-uUZ$854)5!w zCL~00+kJ}I84eKvydZHDlxyH;q&LHzv^+9}UyL0~ZR{S0>3{v9KYc}Rd18(-K0^Ye zUfyF6NxU|}G4(Usg#R|$k-V=EqQU})9X`er3W(~{v>&wHbtt00L z(tSwSY7WlKoqUjaVwlpN6>tN|5U5YISt z1IQ#=eJUURbrqKR)9-Y20O!jm9R;ecq3LPSja^)19(&817Im~1@!=5h6qa<8d)%af z6W9PGL7Xt};hy>6?6{loJ)8}MO>z6-dh2AE;}flHwUQ`mzdsb{Jkg(Mb1Kb(d0h9( zn7o+qFhB4&5q@LR-O!-Z-YEc~w1b|B1!+B4g|O5mG&y2+yi7TDYiaNh#MlMkF)@zO z5Ng#sl_Q9Siep=#!aWggVB2PX?`>$dSYuT~X|kvdYy% z={J<;|0WUv&j8(~^D27zxv3VCAZN8frryy0nz77VAy zc)Teyok9Lf3D38NX)CZ$Qf?}j9ws&_IN8$JPS&!85>BPDpDHCO$cZ8DkB2;L(b2ca8!3wf551gd6 znm_c{pOTQ&kVdG^ONcg2rG~h)PjQ-40VSI85OS&sG28?cIFGEoIM`J&_U~e>Yn+Oq zyHO75b{qAm@(F*Dz@~e}5T>QB*(Ws;p0UkoX43$)F@J{{=_0r^ewGVdQZ&~)ZvDd) zdC5kY&e(})B3ysguwF$Npdu{IyEex5{fe9>97ROh!hyOm$@nN$cONVS(8jVfof1Vr1Q4aXbEAv+kW+*0fw!{HFzRhNUV1U5C^|7A7%M?oi_GCu z6efE?txq?I@4Nzvsn9d%k*OcOI9qs-_$85&59=55$L2CyeC+tScGsKg(hkGMi6}Bs zFm)y9IJBN}LbDIb5Fzn)DKH%vNc5`^l=@>QR7z8xg0_+;hQZ2Bb*I1}Q_Q!#r5G+0 z3&c{iU)frP7p~TrA>&vN2HorFw-6Gpk_O@*5&aU?Q47@NMN%-&v|38t-NTqXs85#Y zfgH`evNPep5rz=GSpN8#>gkCsO0X9ih0P*epk?N%K2u*R3x3(a=NJpt1aCd*pO=W^ z8Q}fp#8!ws<8x_jy&gERu?bed)^wVuxEL#m@z&v`wDc-8VS#2tbfFDF6nE5DtTSDP z?N_H%WNDJL-%W81eu5o6D>jF5UDwT(JQXTzDT6ack(1{qfeI`*%M_%mJENrGQXsc1 zEUB6rqQ^=qGNMI3Lx3}$;>fhBnNXi(_`{eOV*kV-`WO;WTL92Efx<5A8BzJgUZ>*gPSC;YwflykPoSycVdbRArOE+WAN6Ll_@v&tSZWCa zhgrCMw2*QeF$^nsNJZ?Pbretp8L?|Vz@qf7In=hsO^*$)hM=i89BC{8WS@t9sxS-j zPCw|83Ro-nE-9{MB#VX&t!?=>>wHctR%LRQ7J|fw3O0~F4k2u+%-A}a(T9Y|`xUaH z=P}%y6su}v3|PJ4>|(~KaSYw)Jc^$_I;AW^#OC%sa!SWUAN@ zEe#dn+)KY+${O=C*#o|l5?qk53VTn=Ccuh-dp5`0D zLq_H=V+OwrNXCzM@a_$D11&+8ACCz;+M-euo-KMm!V~=Zs;gXGH}up5t)})NIrqv; zj@~mBB%bcT8Rr;GT~skxQDtXB|6zn(MpR48G>rtil@%u|HZsasoq2}PRWYN&k)#kfQh-19{lSSm-pMU3m`C0#+k)+C`qwP@# zX&Ofgk2T`n(nCgMzK$^IW|!v+Cm9_3vs5ufZO=$&8UhTnydZo1jo#$vmKyw23WdOQ z6IW`#Jb@*P=5eWeKJ0$XnC8a($mTud;orrP6eR2K02F$o9ci7Fy)<_9=$0H4u#}vk z$tms*(EL0&G3^MCbL1FB1gB0%jp7qf$uS){TDsxl?PHxTk-=b%j#))|#1fD(1#Op} z<=YZt?o0^=0;BG@sgWf0QZ+X2s?Dnf+g-+^jEqNYdujz;cu$HB!RdZoXS0EvGLO+^ zMfu~L)a4}8f+k3@o}5~~V0K{cKn6xNrg+c9h#O}}&wLw=VwEAo!oDTq){mhU#-=*d z-XpDzAX6ME6RKB;a}I787ne>wD9;Y((gr$IdWgGzL3iaRDY?z z!iSUo_DdzvC^RBq{(zrdtO94E*8UeUex$BK-3sIjn#jo*V&+K7YJGf8FOK}#oswzvQBiiv-%#W^BzU*7FP9V2z)Z|BdIwpjN zVSP`7o;gPU7t9iFN>1((HY(v*l=nwFTN(#M4AwU-8ewQRCE!#W#c<@-Di$%(7xvqg zHZJvPDD2_P0g3Rtg-rWg%1yUH+uTp(OIf14-;D!maY}J*lvSVqQD&${lvL z4dqqCpp3HS0`ek>bbvS&Gc`z3vNoi>AI$t&!)QVuiag@1@x=kZeh%YN#zaMtY`~>5 zo>hbDPaYGhQ(qcOuK-x5b0)l6Srg@FV>#ZlRlItL1D+?$eiP)JE~69X13F&C~B9aNzG zwLuv{lm?NWF17BSu}hI8CiC|{a@;E{*^MHI8&?B`m*#jLi58lVqCqSaySuB+%j?0H z;7|*#e&yQ^b++BFMrA20M_Qpk7!-m%wTo+jsmBs(2ckG<-DJ@#XRnDUnK7 z9HzSzr-6*>C%M_Bx#Fk6`2jM@JJW7d;^$9Zu%j4xQ4xGjo-x+HY}3ZcZ2h$ZGU=vq zE*vzwCEv1ewltB$DO8!x?UzEwQ=gsGVd6D2Y{~UylQ{O}fXmD=NnxYqb$Jo&-wc2E z8V^yai{aa;ZI3_LIRs=D#FTi*4hgelz|&E45^qLn4HiWV!?@k-Z5obf8yRZyIv=@J zAUe}?>E6jYKiUqKZpwjmpvX`~1mndkWeTq9xJ3h3^5KSMp_U#I)W7-2v&R*fLYaUQ z{1Tt1CRu^QX#iro@cWQ0J5>q06Idt7aJo=uZAEsb-Bma^AM4WQT{f_4S#EjDrts_m zdFk&II`_4qBkL5J@nP|nWEyj+45%WRHUquDTo|jkrr*QTUvx?jz1+K&6Vf4R$+e$$ z15-Cp%UrpnlI*b`@<;q&)0ISsrhAj{7vf2|sk2`K-^tl!rLNS^lM)$5p-Hp7;8peq z$pi*gN96H3b|{CyMY!MWh}~I8P~N5R;Z-S3h=VcVT^X9Hic>RqLoOC283=O%5zCh1 z`9Shb16&iF;njUh-<1ryCKf(lA%xFvtVd}t&HJGJ%1E|N{>p0F%=v-xa9faPG9$A& zFa>mx#7=f@l)zzXMR@oqfP}$JbO8^SxaRv(yG-?VQ7c`ai1r=H-qE9RfaJ^1T`IjF z{b0=~O!}6#}(T z$pU?VZH3zaHL#&V$W~$^9xk>`B_anUDXD}k4YECI<0M_VYQbSGT_X4G!O;{P11Vgq1YNA<2I>F3Kb#@@Z6dtf(xOz;rh9 zhe9J>w0C;j(2FxIdOoZTHz^XNE8Mc$Of2h zuU`uKK6=g*Ayg4+Kx%p*sqlf$c|@MTSyeVL5`9-Cxd`Dr%vMhD<{4>c&S!vZOj;G+ zrsjv%(~aEp*R{?G`;4baHO^hGKB>~(m(!sqh2FZ)^Q(g(6fmoP9HMXiBnk_I?jI2H zgzt2ED~eS2wF&WhB%}rxyBf*5&+nK1_Fl5w9e3`DIR(?oXQzwsZw69zSO{SXQDxl? zj$lmj{YW=Gt%&&sAG<2YJI~d~Shv&Kb=vK|ET-JJJ0;ZSi}Sa}+)4P~9t;LS z77E~(Dd}b@A|nY@bf%tC{$?5r9OyjTO*B9BzkS|j47_tx<(gZ+#kta302*G091UUL z3HFy3IU}jI7Dw)*QTFK#ROP|Ev)_fdu;H#MxvsC8Xu~U^NT*rH2*;g9cxS!+?PE`o zNHxGnmU5LkmS@O(E10l$&=^S5@p2xDK}R};gfDY&%j6RzIxABsEdmiTBI(zvp(4Z+%~BZH1d~s$xRy?6 zRv1QWD)BfZ4eoQ{%hrin;ZC=BNFe$%>ZSxkscO8D_#^E!qp*`LsyH~*HZX2c|5<4Z zr);qRQJp6}ecg+qb5+*NhLH;H!?`3K8tOm#u`iru63<`~m3E}TAa`+~2SpiGD-nDk zD#sy}UEb}A!CyzVC0hU8{EqM#X`sM#xKan!y^1d>>xkCFQ8{(~yfs)2r|(|D9^Ed{ z4;O84BGqM60xdJPiX;Ch$;i^ ztv8*F5nRGnO6@Br+(cMW|vplo2 z5H+`FRqrHM|5Gc*4#ODccBSxO3f?LTFFD{p*HyHsqG<5fMc~dH3iji(suMoL^rayW z|AuJF0&Eq-g6EmQt_ioTR3}wjk_n>bftiGd?VN{j{7HbDui+PZ4JTzNE^97Ov;$!$ylg$MeLO-|jsmY8d# zV)RRA=@^Jhg~p1iW=YdOTNqBzHyWrLPP45oL*T6Gcp`=Oy9H~7@lSMJVyibS+i5dp zgO8$TUa=F;kGsxAd!sr2ufv%tHe!?RA2VwwJ`c#q^4A8=$dM_GHgm?S?-}TJK4+ID zruD=QCYRsx>(4|Fg(kO&%|k_}3i_K-0Hf0K#7S^MfsZ`53+)+L%j@|LH~8Ka8u00S zzR>br0}5}P(3Ld1(H68szH!kzovA9iyC$UDs9St!0E&LAw-b#>aI~eIs({Q8o=w^W zO8>Mc7I-ktq-7AH8pqYXzD3+)GD9K6{FLG1oH$Y2o~Na`&+U0#KdxrvVD@t<0D%MN z)Im>`x-(W87hsr58E0FegX(o6#=L(zIlfn3X^pbcErY5qgAc8*YK4@s-mcBMm5mUH zx-AtyjFG`iV zPfXjAX`nZG)8Yj)Y(Hc8_uY3aagHi)4sY9sd|05Kg<<&q*-}rIkSRM=X1o3VmgzJEx_0+l|5% zDr&AMgMPfOU#ed5u(BX9p!QW;INsy!4D745rzCzZ*sUS@%hy4Y)&@1Bpi&aCqqEo0 zt*7aa^jN7*PC%-?Uyh4*yhVNrQMt{Y)b`3x8qQ*4uADra6MR_6$*3tcd;N8Tp|*Gq zQu?@pO0ES#aQ16z;~X@u%@$s{)vhzLUJPtMztdD{)|4_*YNE*i{DqC7-GE`)N|~!`iTz^IxqlrH|HWi}Z?ucE;gtl9I}$zWj<+PIn=+u@qKa8g=1($y2TrQkPY!?q0cK1M z1Jx@#wv?t(YCap20CIbDw??naFE+nSYf^RtAxLomPe_b$Wd%oJv5Q;SBnY-Ah6>Y* z-&S-2N<|(YiAiOv6>d+*P-8^4xtC-rQwppgU`bFw@Q*YJkAd2KH_*CY9tE-1xqiWDl&Lc z>BIWEsE=($@TqMGc@peaO6_Sifv~QE{#4~iC~vXQho$`Gd4<7QiAmYV$+aU7 zNSo&^Mcs4Lq7M{>V2g|hPyv^=8gZTFxo9daxhN@X*%NaP6!jV>+LD_;Bp9{h!_&14 zH4~h-q`euD*WAw^$&n^li{xte>kR`%!lT^lc(8*^?P^bBueoq0YV5BggnJ3tJ~vb| z31#R%8?6pQr)yK2-y;7ySdtSdMOflRY|+D2gY1kcw?dFkCGaB)8t2{f@06~!wb71D z#?_S**!RDr4u)FbukR3?;QP4s7wN|gw8o_FAiLyka{Gdf&UVZn-y6H<#oUJTHbzmz zFBsI(hP1~&{DX16 zt7wTX1Ba~k*5x4dOWoh?SQ2s^T;QN}l5b|lanh>y8+;i0Ph8BND2;T8qdQbq_z=)Y z0cf4|v1+td&APy^6*Uz?fwU3Pn2MVhKzlvLw93zA+S4WhUER|P<1;F&E*8jBp6nkG zM96_jFIIUwnEIr78j>njoS}|r3#wh+t)V_lg@+Bh$$4o4)oxg#R3ao-YwI&IR53Rf zSVd*&`DZ$-PT0~0lDiF4lDqy@8mjC%T*P45z?M&Jp(F|Y^9*r!tSkd%&C7gQ98sw$ zPn%6E5J0LJAbCQQVKB(FbdLyc-arxz6Vr9LRjKAF9Ej%vm2Lt+#}NsFxvB?>6EJy;QYtym;>Yx*bHRw>fxcnXtY|i&ooQ= zuIjjS`N=9*{8E5=aZw}-JQ+?Q?(G#RTR@_Es&mcJn6m)aae{5Zf31Y)elzwBS448k_RQUkeN*#L89z{;}^F% zp9K|_7qp-qa28`_0`5T2J8%E&2bLH5G&!D*D>2Xu?(Bm^_{z6Yz8);VK8k3`P4%C% zjQnWo*hvI+e#Bp?+=D032_%E?iZo44Tp%dRGuk9oJUVK6U1&iQoz-|Xk{=~@k_1u^ z;*1#XA5dna(vRhAUSOC2ra$^f4phx`MRARE7Y#m0&^OcKSbr`u|9n^BASh!tu9cTiX;BRL>xS*0Dk z{GIxP9@tg((J-NP<3Y;p1WF^ z4jHb1>&P8d%%nCAX)41(f|Gox1gzt(*>r}KfAHKtV3M8U9}Jg>p0}y+KlY83AyHbR3W~($xX+C7RV`GSsFzc~aw7JsqLTSr|A>v z(5GYlsggH1Oq|L*V!_z5#2cmo&Dc^}I;OA@+TOWJoRqKQR{tLdvKf7yK|APgh@IxjQ z@dEw5**m`Lr&$aO@s-!}X?ezN4~`uCR|ue00Kvb$**p%Q`|lV*?LsIS|F%uf4G3^l zT3|bpCGwv^Vv!+(j2+6io<=aako;DtsTl+}D=8Rx?S`QXUWuCG<92xOyxh);K$x_9 z2yz3#ww$gWrrK=%rRe@Mg^qGjpS$T--V-{fQA_ovwa2qV`1d{$k3(Nr=7+X`W$_92vj;GyMVuKR(9 z`PY_Er@{WEZ^;S%cO@~*jR=e9wDM*D#W2kkf);zxHpghBS$pp0|FJiA?3G1*e>X29 zGv}rZFQm!*?jQyRWow?&U1mau;BCO@AiAIVeriLubL*brV}7M@y>bh48yJC zf7}P+Ak=DTKOSBul>J?ob*(+N3*kw<$m7v$rp5diw<s2gtoA)9iG`V58W=zeSyZG`iqlb?= zUQ3$Vn)~uzrDD8YO9aTllTGU07wMo$gi!ZX)&1Oz{||HZ_i6NEtzVdxSd3@&<{?MG=N z2o#LP+<8)*mvXrA-ylAU0}=-}v+eQJd82+&>D2mGq>`dO{THJ6A#{>qQ;7dwKcS@I(g0!UMQW7HJe^!6@gZt@Rd(E6ZbI$qX z`!zkafphhLE)MPh>81|-z92q2YPw*EG^NbMj8AB?96CD>0Dt($1^5G~Mqmb@_=ZZLj-Z2q zWp9v3eAm6d0mkEuKYUzRArLHi2T1<)jUYH#V~Zgwtn!NjZMEa{Xvht~LB{I{S3!cP`R@tE?Iy{ z^>-$f@dJ~dM*B*j2-eQHWmggn`s>S9y%(O_8zX4k+QVFN_e9JuR%FA}RLHx-nn?+{d z#ETX7U%+WcMIJO zii>@5x=(&g@_x<0AtA_y3@)oRG-X6(8>i9Ei0o5wb+tO?`>Z(NMm?^4>-Cx99ThI( z5*!qUY_{(t9@qL%eDgRJp`jVR(Nq5c$65SE3=k~#LVA9S1DaURTbkGa7LlnR=!OPA z9&l(fPgM?_l#32r@Ebi^o*Nrp35XhRV~zp>Uof8kH%QoH`9j*e_U_(2PFKrT4op6t zE#d>awE2iM)-EX#Vlv&F)CA7csoX_Dv~uC-)whA0=V1=M<_q~y1hblg8UM%?AZNm#JSe&7Ibco%y!)kRh7SMc(+CU72BBx|FHyVp(lFW1WoczYGm`{p z0PFTN(#mLHmc^>Ey>vm>ua3J>KXYaRqnfIT0v+qhueHAy#8k%#TS!El6t5VG)}GpI z2WiqWgQb+G`O=UW6hpP-l?RelKAdEo%^$y%#EvBF%aB%VbC0cA0deBg_T?2P0UlMq>dJRuIT5RtT?VeDl2i~uKPb6;A{WbPE^o`r!0UxNsZ``v2 z_&PD>)6DCD`dx--wWE52aQICLw1CdCqB$>FIrFn?f%l?29cdv&R)X{gQ2dcO(*C)K zM*q0y%1?lj@WF_n;=Q5Bd_hUFRRhHms?1xHsy&4RM^ke3|GC{o(x!bvGG3vqF5_;1 zuSx8}=5vCZ901srsTIWn4B~YussI!v6&JTsQ?H}ENg=X3Y3uAg;nsG`a5g?s^%26I z1;CW5)q9rD0Ir>&{zn`e^ILVr!K+CvPh24ulxTk+W^=5LN*}nF2^jCWPtxuWG7F#V znIr8t{#g+_Vls)+2%0wu)8(O#(x*48uKR76e6W)9p?%9xKYK^FyY->o0rJKhue%?%EUe5UrS~ zU?PJPp1%v3h$ByB(wa9c*M$4Tj4};V?niW~E=`{W0k;+ki^Uxh0e}ZmHw{UzN|-VY z1_-vWBjjcPkc*~xDPvL?QOvQN$7y36lK#j0gX4NycJ@MoIXgx4=F7{-^!McSOzV7g z-$H`Fh755%$fQ+`x7IyQ4Ezmx`d@2VTqR!ur+_G-82thv-ra-Ob_dzfQ2iFK#E4M+ z@2>oL>q2^yD{Z7or8N`}cXU2p;_|p8<9K5!YI!D3CIsdC3{)sy%$X>g_SlR*xz0aDWSee zu15q<0jCd@%`Q@qS%WqqIU*XJZjMzu62ZYQgPLO`Nj_k7zfz168dKB>Ly^;<&`-J-wLrF~ znWuY&Wk#sFggJk67YSNc+={ABZRq#AGG15Xj_`g}{|{+om%QQ6c8+0Zt7O$O zIaRO{`zk+#urY&(Ov6r>U^`|`Etr5ah497YKd)LB3rY=Hk$8JGrzk7QTFD})p8llS zlj(+updMzC&qkZdV`Sz{j! zrzOAD3^Ji61jo8+B1QFo?^jVpjxMK;`^@og1wi4O1e0?trMaXg=}mMm4p|8WDm85c z?6ZceQmD_X^WUK;Rx!{a#p&wix?Q#?)eUyd%#OK4B}=5rZmWJe5~e*it<<(>CA70h zQxDeGRQP5jlTxk0gY+&KbVAkiWJnYimY2M-YZb1-HlZJDf|@3Ji^ zn=%ik6(Xkt;eR0z9&B!vg(gbc>rQ>cvnBIa!l1>TjN*60^ul4M0AA!r_7`LbK*T6y zEOr45f3efB@$0vh#&e3oR(5S-hDQIah&L8=*TF}Q9l@^8y0DUFK(A6lHmvl4QEsK; zRqM0pB1zrarFOPdrBv3bjw}Hv9V^!dI$S>8o7oTmbtL_X zN)3U~6L;L3->2ikezsNFFl!TflB*30Rd_lqwB2Ol50Trd*a*uuOvho0*$$Orm6lsxu1ntFs3wqCnHzqLOyqQe({{tJg@=mMmDWX_l-{@Gkw6)(B=bY$~uab zx!s!ii^Ds1%n*&>LFDHU$c>M-(f@+$G}+J;Ib}3)^U7fG^Hr8A(V0)!7&lrhP1VN| zXKyvWj+Im_YNJgi|0$15-?ct%jF>wD;UJFCiF#f!eIU{e{xwL0Ti#?inXFA)b1l0u z_Qnii!SnaOj;6Vx!)S5_px^o~ja7X9!ynbJ!11ecMdnsfrLrAmX&q7}cYDJ9Zu(wx zYlq@67;v4ih*>tuGL0r?sBiMr4Fj0rDfm<*1~p)a6T(atjy=btLKuDTgVNVNs8 zkvyLO6Lroa*%2IG5+UNMt>04(+vcB|#u9GV{lr1p{z{E_ukir;TrG$-)%S1LaDaGH z!Q{~h!tpJ2g%MvI%qLwr#kpIj!Mp56(u;AzP%s5YT5Mm~`H*+>OdJIKsU zM8~v)B&U%|OJRjZT}A;>|I9(9Ch!jUY$h64i#3|5>kCGr@Y@1s6Gf9-uUMfe`sVMv zuZw!KdaugVj|>4neh{aE(S_8FcJAdj6Kk5WR2446dEo#c;ODv|;w=Ev_HQjZv<3Na zS7pxg!3kyn>M9$FAvXOaeHthqJ|VumChm;!BR$^$81eK2+;=+! z73$hwwa=6edEo}ajWRRbvt090p7^36Rp2Oe)yb&-CKP#lzc4rmS8{7d8)Ek_HdPfy zRUib3G)c$jUb{S(vUyV$vvX|Murw{-kwkHG2Edq@$T@bsWI+>3*HHfH%HJ!=6^O=4 z%XFK`OF&#{5|*2#X~1>+JvjL3m5pIE;gVEzY2PEv^^;pP($9qmP1;*+J*j5x^6%Fl zeLHSk0NzV+sZx(hhHx4_wL~c+CU5Ph*d_M)q$HFEst)yLiqiXIAm!k8Se6BMK|vC6 zgZq+~DM{ThXIBQ|_kbU~r+bU&ufM7hTiVdZ$ajh6QB7)EojS;IWP()J-gnx2f687{ zFLS$RKjB{1A5x1atD>-WL`nTt^7@0uayr!%)6>Eb)rz3~pj`n{NpyMyW3OO?w$Qj2c42mZ>beRHknzg)P zv;Ief4a;E@dhGs;1{=<*D;03-et1 zo=uomMgIYvEtS}8S;oix8Cn0l+Km6&1S6956CKJjBGdDA1`n2ub5i2(h-fUL@{S0a z_A-spAdN%kRRv00IYJ;!Rs>wr3!kUE{j6+4x36Dm)hNq091-jaJtjcJRoSj+q#yAZ z<#D*G3ILHA41kCAI{~Rbnp!(zGw%gmQd2Lja-0q9b@P3FcE;dDd{hOEdy4&Z+|5dn z75y2LcPed=)o%@3QIrbE+r*;$o>8y{arYOHF@T6=6o-^!Cn*KOZD8UoVOe!z) zhRw7@A_q>No9G4AikmhJwv5vxpClfP+}PMaER2ZjAG3l|xv>`_e+sliJwgp_Qsj{s z?lSqE-B|PVa+>n|0A`=O680-!*Ww0%#3cF@e?O2;K`6?(ixx1{Z zIVua#soX+vKAX($n2fWEoHw>SN2Bd`{pNj2=D;-lYZ&l(q67@B!S=q3!-wwf9T%HY z9V>>4HO%aF-5xDISP&^tyxsgVz&^Fvs8*mRUigtL7jOg}UeI}XL$OF=UEI5cg?Cd6 zdav}NZ7v3doT@I!sa4=ESQIR|U%xlHSgNauD>=2=?1NP@qQZ+6{pZ9~PAz8pd7f=s zN@U-}^v%7=eIPH&J|1&cx9MbdZ*B6+=Cq_|hK`paghD-Athw4DZ4iW8CCVYZ#p)cM zz*;6>8qaU0z8%dKrTgT)8~(>B;cd%ylqo>nLDKc7#ptW2pYLzQxuh1F7*!M|aRc|T z1oU$&=2Nk;L*OsY{lLlG68em4UjLIHg86?sh*J^Oiugn9MA_^dXqk!4jPLajCbkc1 zQ4^r33FO+98C;38d3xhH!XSL&&0@~}p$*}YrKeb{{#aH!foaUSd}9cgRDV)uM2J{Y zg27z&{aIgOp}L8O`nl~iu}1XTicaMDqva)fX00o~-PGTp0g2{QWw1GYRW{M6K2%Vf z-l0+JB%Wb9;3g!&sWsEg|4?DxQu0mB)r3#m2nu-)ads-wsY`guNnr7QT^URC1p-Da z=SRg_CbIsSDh;Z$HKPQ56Zgn$0Ej2J4M*k?u_uXNjse-I+9S#>&L~~QiOf_Zkvq@7 zPaj`j6FRu0NZ9Dj-1%mkPbAyJz;v-v7r4i+lw6zOo0!CJQBXzlyLu6&z?w=}T6@T; z&$Mx=cbT8Cja_huB^n7=QT%4fx_}y+uNZ@dJeqHQbR<^X#e6@7R17qp30J(56$Nv2tJh6BslQ2%fz^QAatoYz{K&nf~ z=wl@M!N2$JhZ6@2LmE;6$if~Y^AzS9EcT(=&6Fp$NWmJ@u0C9OQ%|;gN6}hm6=L-# zRqHcm3`vpa9e<#4Ca#I~XD7UfpV>s_5{t^daN<~=@7jRSk_EE3XgxONY|uUI+gaBi zyQSqRN#nm!ms9?#vqvsTXT(j$Ua+2Wo*o^PRi~~P)p_dmi_r8NWerjgIjx$j3dFjB z08@_C%+Io%*#ki+8W;3a7gCgh)D84@_b2_5fAj<(KRhUA!H?HI7f1tRGTSz_Lv1i1 zr?f>>4^Rt7az<{~JT>}K2*toss@8JtH92GsPW{Caj4xe0Vwp1gB|YN3MMKxQ{SlSn ziD_Z}juc#S5g>iQTPjeXDkYX(pQcyR&p=h81-H4^G+KPdxYs&B%2FG0&HGr4;sPIkjv(HJQUGOF{&1=r( z+!CeadG>GoA#Zz})G~qY-%x@k?fHOged6%+h{tv46fhd|1d z*_7v#NCN9HDjoyOmXfzd>{h-Fl^tovKmy45T^_GeC@yio^(V?TGL--F5u*Jhd14QT zc0_xpc_rjTyVyV$n7MJOz5S(yfN`xzOArat*DyZOw8pCwt|1{sq31p>2{(R;8I9F( zM?Wm#)so67CLcCbLt5*>Y4}8?j|I(E#RNm`((32X-}y*E%!U3F5L`e!SfkQ2aUULO zW11J_7Atu{P!Hrkvc{$nb8W^(!6Idg6omX!?Ezv%sK+%e=OQKTxoHS^YIA&ebXh$y>*Fo6L@J#Q-z`L94c_m zLpHgD_ZIS+rR}%`(_Cb=})q?7Z8oEF0BccHB-bo%pA1cG2OVmuoFrc7XW(0Y3&ULCp)PVEo+=SQgFvY~@(bukR_3gN2ufvNa zRm)tq*k^6MbKU@%Br9r(hMd&9p1M+dv0{z^BvPS{UcCyp`zcx?gWUNYW+SVY1obMS z2GP@>OI~kFd@AG-&k$uvY;&gw$k(4&`>mQrjDb~ung<{ja+T2>=Nplnb&>5Fh(wB`G~ zc~xMxS7eOMsT8H-2fr*pH}7}!XvWdr6gW5=j9>#Xu3ZIGwsdJrBt4+s@0>%E6PxY_ z%PahJtn(NBTR!;GkWpX?>>Oo2Pwar3y;UgQ|cHveu5+SrhkS z7N85^l#@pDLC)H4u2k)blt9P`M_}l_YV>U2kG5?XogBfwfCZ+(m9r)Ql%zQQWW5*PIf zZzPh)JY8wCrf`ZQfc34P{K6}-z?llU%(FHiq4}XZISEnW9M(}X4gj*9-jXb(rpSCj zyYO|?0n~rax!cYpaaU+jjV2N-?0o2HV`m>qA9$}mi*m5JPdLXHyzp%ISHXMrPW<2i zg~~RSuOLc1duDtM?ALN(ZM1M$lW3f=8O%LQcvUe%dsb@E|bSPz5FqQ zS%L5ldZBv6S5dAKmmU7&#EV~Ds0VduS^9*uxMbk9Rf)(6+$C3}a))!hb+LwqW#mY; zcRFO(B-zpBT4dQ&DO_>|J z5pL`^cWsE+RYx2pMKDTf)fhV^u8!ILBa_bPQdMna^k*P_oLKJ*q{k83pQ`!%@ z07P<$W~yQQ#f~GrG`R_C4B;HZBHV_?xmYjdFtVxntLxO8*MmTlNX_$TOH2CUAu|GW zW-8w`_Xa7p)H@Y7aiCUo*bo6@yAblKPZywiwiKe8%vx$Q-fsZ0&`C)Zwyo(hTlh}R zLEqwf*lWl?3}b%|3Y@hNiKB5?5N1w=#VTy^G;(8_H(Z2`_<&sbG5i(iu{u zN&7HAIA%)Ug&nc6us3d6ekMYWoS>$i=w30Y6%&p+FjPOzocDF2d6?hod@nJGeDOP@%wFVU#c z@>gc7W3h}eBS^Ys;GzxZ<8@qB(4i@j#n^JYpolsG2gc|(~; z=X%o5t#(pXA#1$DyNEvric(e~*v0ir<`nT(qI-vfX(5)WHw!5e;SM&|)_fl7@Mael zn;GRaq#a;XpdHEN!y6vekVi>;1?k938i&~g#&pXRFS>TPgI_pFTOlBz zvcta-&^e8Mn(sBQ^qX35>il~w5q%dOxSSjhXq-tL^_?;Ux^cshZsa`ZkJaucd@Au? zh<~-Kj{aI<;^%k8I;+4J4C8guZ$I8Y8UOJ_@e#YarZF!4t+>CKM}kuz{a$;}@`*mg zVwQLlwv8fZzjpOQ?zxY$Sn2(W3$p^>e`=b0F^U-LTRLM1^3h@|9_5y_Y0=^-U*Fch zbGCmjsRIYYSIwj(C}CkgLluoNgymxFzH!HubI2-}?Hn^C(m&WB1?PFBO-dtBR z@TwaFS}ax#`QLtIvv?2_4qL*vz3_I5KP26cnNS$=LegLz+?!t)hfhZ;z z01XB;*()IZ>1~9}QwuQO5=F0pw2Ba`ViV`2s z(zVsyD;EKJFGl5>#AcxW1*%%v)$7;bv@PgLK$qALY-`5lHa3Qd1^~a#XAIBsx!{6O zg-vpI*v;1#cLJz>v*pD5D{o^1ePs&Q0$mqRZ172Bg&;s=X#b--)BLSuH|b`d0HbN4 z?x<1V6WZ2tRsQkc$9C+D+Y@%Pi#Vh0<==<-h|zgsJRN#Plmvn+uNep#|Ge9 z#}SBFtrk3;MmgKo=xNJLpJ$o*J2+UaIqb3S^uFiE2Ej?fBZD}tD~{~yq79~eMteNq zrDZ>A#y=ZC-LKT1`}_PcHWUkZj^dt*tr$?hmJS6d`JPDzbMkUyT zl?G_FYy2^Q22ha*8m7E-Vlrne;qp;5xhj1&wLF5=So4!$u4)m0sC~7lR_T%8)7Eso zqXJ*qkFCp1rTtV|7CE+=?ptX5W@1ndm>J_I75a1T>yTPo5FXC znQ%rr^PUOSze4&WcFz|s&qNH=0imwNB|9-DyfdIzbE5$wk^fq^n)dG(Eg0kwC4~W6 zzK}q>caK|Y$bHqClr6f=M{I^GwWbg2+Wm>lZKDFELfcMBrH2uV(jTD8SgP2lA@p@$ zk0v(zJ3OXVhmz^UL}Q5&f*/dev/null + then + return + fi + + id=`_gen_ChangeId` + perl -e ' + $MSG = shift; + $id = shift; + $CHANGE_ID_AFTER = shift; + + undef $/; + open(I, $MSG); $_ = ; close I; + s|^diff --git a/.*||ms; + s|^#.*$||mg; + exit unless $_; + + @message = split /\n/; + $haveFooter = 0; + $startFooter = @message; + for($line = @message - 1; $line >= 0; $line--) { + $_ = $message[$line]; + + if (/^[a-zA-Z0-9-]+:/ && !m,^[a-z0-9-]+://,) { + $haveFooter++; + next; + } + next if /^[ []/; + $startFooter = $line if ($haveFooter && /^\r?$/); + last; + } + + @footer = @message[$startFooter+1..@message]; + @message = @message[0..$startFooter]; + push(@footer, "") unless @footer; + + for ($line = 0; $line < @footer; $line++) { + $_ = $footer[$line]; + next if /^($CHANGE_ID_AFTER):/i; + last; + } + splice(@footer, $line, 0, "Change-Id: I$id"); + + $_ = join("\n", @message, @footer); + open(O, ">$MSG"); print O; close O; + ' "$MSG" "$id" "$CHANGE_ID_AFTER" +} +_gen_ChangeIdInput() { + echo "tree `git write-tree`" + if parent=`git rev-parse HEAD^0 2>/dev/null` + then + echo "parent $parent" + fi + echo "author `git var GIT_AUTHOR_IDENT`" + echo "committer `git var GIT_COMMITTER_IDENT`" + echo + printf '%s' "$clean_message" +} +_gen_ChangeId() { + _gen_ChangeIdInput | + git hash-object -t commit --stdin +} + + +add_ChangeId diff --git a/extern/re2/libre2.symbols b/extern/re2/libre2.symbols new file mode 100644 index 0000000000..8308b64892 --- /dev/null +++ b/extern/re2/libre2.symbols @@ -0,0 +1,16 @@ +{ + global: + # re2::RE2* + _ZN3re23RE2*; + _ZNK3re23RE2*; + # re2::StringPiece* + _ZN3re211StringPiece*; + _ZNK3re211StringPiece*; + # re2::operator<<* + _ZN3re2ls*; + # re2::FilteredRE2* + _ZN3re211FilteredRE2*; + _ZNK3re211FilteredRE2*; + local: + *; +}; diff --git a/extern/re2/libre2.symbols.darwin b/extern/re2/libre2.symbols.darwin new file mode 100644 index 0000000000..31e8c52209 --- /dev/null +++ b/extern/re2/libre2.symbols.darwin @@ -0,0 +1,12 @@ +# Linker doesn't like these unmangled: +# re2::RE2* +__ZN3re23RE2* +__ZNK3re23RE2* +# re2::StringPiece* +__ZN3re211StringPiece* +__ZNK3re211StringPiece* +# re2::operator<<* +__ZN3re2ls* +# re2::FilteredRE2* +__ZN3re211FilteredRE2* +__ZNK3re211FilteredRE2* diff --git a/extern/re2/re2.pc b/extern/re2/re2.pc new file mode 100644 index 0000000000..d66cf5199c --- /dev/null +++ b/extern/re2/re2.pc @@ -0,0 +1,10 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +includedir=@includedir@ +libdir=@libdir@ + +Name: re2 +Description: RE2 is a fast, safe, thread-friendly regular expression engine. +Version: 0.0.0 +Cflags: -std=c++11 -pthread -I${includedir} +Libs: -pthread -L${libdir} -lre2 diff --git a/extern/re2/re2/bitmap256.h b/extern/re2/re2/bitmap256.h new file mode 100644 index 0000000000..f649b4ccca --- /dev/null +++ b/extern/re2/re2/bitmap256.h @@ -0,0 +1,118 @@ +// Copyright 2016 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_BITMAP256_H_ +#define RE2_BITMAP256_H_ + +#ifdef _MSC_VER +#include +#endif +#include +#include + +#include "util/util.h" +#include "util/logging.h" + +namespace re2 { + +class Bitmap256 { + public: + Bitmap256() { + Clear(); + } + + // Clears all of the bits. + void Clear() { + memset(words_, 0, sizeof words_); + } + + // Tests the bit with index c. + bool Test(int c) const { + DCHECK_GE(c, 0); + DCHECK_LE(c, 255); + + return (words_[c / 64] & (1ULL << (c % 64))) != 0; + } + + // Sets the bit with index c. + void Set(int c) { + DCHECK_GE(c, 0); + DCHECK_LE(c, 255); + + words_[c / 64] |= (1ULL << (c % 64)); + } + + // Finds the next non-zero bit with index >= c. + // Returns -1 if no such bit exists. + int FindNextSetBit(int c) const; + + private: + // Finds the least significant non-zero bit in n. + static int FindLSBSet(uint64_t n) { + DCHECK_NE(n, 0); + +#if defined(__GNUC__) + return __builtin_ctzll(n); +#elif defined(_MSC_VER) && defined(_M_X64) + unsigned long c; + _BitScanForward64(&c, n); + return static_cast(c); +#elif defined(_MSC_VER) && defined(_M_IX86) + unsigned long c; + if (static_cast(n) != 0) { + _BitScanForward(&c, static_cast(n)); + return static_cast(c); + } else { + _BitScanForward(&c, static_cast(n >> 32)); + return static_cast(c) + 32; + } +#else + int c = 63; + for (int shift = 1 << 5; shift != 0; shift >>= 1) { + uint64_t word = n << shift; + if (word != 0) { + n = word; + c -= shift; + } + } + return c; +#endif + } + + uint64_t words_[4]; +}; + +int Bitmap256::FindNextSetBit(int c) const { + DCHECK_GE(c, 0); + DCHECK_LE(c, 255); + + // Check the word that contains the bit. Mask out any lower bits. + int i = c / 64; + uint64_t word = words_[i] & (~0ULL << (c % 64)); + if (word != 0) + return (i * 64) + FindLSBSet(word); + + // Check any following words. + i++; + switch (i) { + case 1: + if (words_[1] != 0) + return (1 * 64) + FindLSBSet(words_[1]); + FALLTHROUGH_INTENDED; + case 2: + if (words_[2] != 0) + return (2 * 64) + FindLSBSet(words_[2]); + FALLTHROUGH_INTENDED; + case 3: + if (words_[3] != 0) + return (3 * 64) + FindLSBSet(words_[3]); + FALLTHROUGH_INTENDED; + default: + return -1; + } +} + +} // namespace re2 + +#endif // RE2_BITMAP256_H_ diff --git a/extern/re2/re2/bitstate.cc b/extern/re2/re2/bitstate.cc new file mode 100644 index 0000000000..6f045b19dd --- /dev/null +++ b/extern/re2/re2/bitstate.cc @@ -0,0 +1,378 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tested by search_test.cc, exhaustive_test.cc, tester.cc + +// Prog::SearchBitState is a regular expression search with submatch +// tracking for small regular expressions and texts. Similarly to +// testing/backtrack.cc, it allocates a bitmap with (count of +// lists) * (length of prog) bits to make sure it never explores the +// same (instruction list, character position) multiple times. This +// limits the search to run in time linear in the length of the text. +// +// Unlike testing/backtrack.cc, SearchBitState is not recursive +// on the text. +// +// SearchBitState is a fast replacement for the NFA code on small +// regexps and texts when SearchOnePass cannot be used. + +#include +#include +#include +#include +#include + +#include "util/logging.h" +#include "util/pod_array.h" +#include "re2/prog.h" +#include "re2/regexp.h" + +namespace re2 { + +struct Job { + int id; + int rle; // run length encoding + const char* p; +}; + +class BitState { + public: + explicit BitState(Prog* prog); + + // The usual Search prototype. + // Can only call Search once per BitState. + bool Search(const StringPiece& text, const StringPiece& context, + bool anchored, bool longest, + StringPiece* submatch, int nsubmatch); + + private: + inline bool ShouldVisit(int id, const char* p); + void Push(int id, const char* p); + void GrowStack(); + bool TrySearch(int id, const char* p); + + // Search parameters + Prog* prog_; // program being run + StringPiece text_; // text being searched + StringPiece context_; // greater context of text being searched + bool anchored_; // whether search is anchored at text.begin() + bool longest_; // whether search wants leftmost-longest match + bool endmatch_; // whether match must end at text.end() + StringPiece* submatch_; // submatches to fill in + int nsubmatch_; // # of submatches to fill in + + // Search state + static const int VisitedBits = 32; + PODArray visited_; // bitmap: (list ID, char*) pairs visited + PODArray cap_; // capture registers + PODArray job_; // stack of text positions to explore + int njob_; // stack size +}; + +BitState::BitState(Prog* prog) + : prog_(prog), + anchored_(false), + longest_(false), + endmatch_(false), + submatch_(NULL), + nsubmatch_(0), + njob_(0) { +} + +// Given id, which *must* be a list head, we can look up its list ID. +// Then the question is: Should the search visit the (list ID, p) pair? +// If so, remember that it was visited so that the next time, +// we don't repeat the visit. +bool BitState::ShouldVisit(int id, const char* p) { + int n = prog_->list_heads()[id] * static_cast(text_.size()+1) + + static_cast(p-text_.begin()); + if (visited_[n/VisitedBits] & (1 << (n & (VisitedBits-1)))) + return false; + visited_[n/VisitedBits] |= 1 << (n & (VisitedBits-1)); + return true; +} + +// Grow the stack. +void BitState::GrowStack() { + PODArray tmp(2*job_.size()); + memmove(tmp.data(), job_.data(), njob_*sizeof job_[0]); + job_ = std::move(tmp); +} + +// Push (id, p) onto the stack, growing it if necessary. +void BitState::Push(int id, const char* p) { + if (njob_ >= job_.size()) { + GrowStack(); + if (njob_ >= job_.size()) { + LOG(DFATAL) << "GrowStack() failed: " + << "njob_ = " << njob_ << ", " + << "job_.size() = " << job_.size(); + return; + } + } + + // If id < 0, it's undoing a Capture, + // so we mustn't interfere with that. + if (id >= 0 && njob_ > 0) { + Job* top = &job_[njob_-1]; + if (id == top->id && + p == top->p + top->rle + 1 && + top->rle < std::numeric_limits::max()) { + ++top->rle; + return; + } + } + + Job* top = &job_[njob_++]; + top->id = id; + top->rle = 0; + top->p = p; +} + +// Try a search from instruction id0 in state p0. +// Return whether it succeeded. +bool BitState::TrySearch(int id0, const char* p0) { + bool matched = false; + const char* end = text_.end(); + njob_ = 0; + // Push() no longer checks ShouldVisit(), + // so we must perform the check ourselves. + if (ShouldVisit(id0, p0)) + Push(id0, p0); + while (njob_ > 0) { + // Pop job off stack. + --njob_; + int id = job_[njob_].id; + int& rle = job_[njob_].rle; + const char* p = job_[njob_].p; + + if (id < 0) { + // Undo the Capture. + cap_[prog_->inst(-id)->cap()] = p; + continue; + } + + if (rle > 0) { + p += rle; + // Revivify job on stack. + --rle; + ++njob_; + } + + Loop: + // Visit id, p. + Prog::Inst* ip = prog_->inst(id); + switch (ip->opcode()) { + default: + LOG(DFATAL) << "Unexpected opcode: " << ip->opcode(); + return false; + + case kInstFail: + break; + + case kInstAltMatch: + if (ip->greedy(prog_)) { + // out1 is the Match instruction. + id = ip->out1(); + p = end; + goto Loop; + } + if (longest_) { + // ip must be non-greedy... + // out is the Match instruction. + id = ip->out(); + p = end; + goto Loop; + } + goto Next; + + case kInstByteRange: { + int c = -1; + if (p < end) + c = *p & 0xFF; + if (!ip->Matches(c)) + goto Next; + + if (ip->hint() != 0) + Push(id+ip->hint(), p); // try the next when we're done + id = ip->out(); + p++; + goto CheckAndLoop; + } + + case kInstCapture: + if (!ip->last()) + Push(id+1, p); // try the next when we're done + + if (0 <= ip->cap() && ip->cap() < cap_.size()) { + // Capture p to register, but save old value first. + Push(-id, cap_[ip->cap()]); // undo when we're done + cap_[ip->cap()] = p; + } + + id = ip->out(); + goto CheckAndLoop; + + case kInstEmptyWidth: + if (ip->empty() & ~Prog::EmptyFlags(context_, p)) + goto Next; + + if (!ip->last()) + Push(id+1, p); // try the next when we're done + id = ip->out(); + goto CheckAndLoop; + + case kInstNop: + if (!ip->last()) + Push(id+1, p); // try the next when we're done + id = ip->out(); + + CheckAndLoop: + // Sanity check: id is the head of its list, which must + // be the case if id-1 is the last of *its* list. :) + DCHECK(id == 0 || prog_->inst(id-1)->last()); + if (ShouldVisit(id, p)) + goto Loop; + break; + + case kInstMatch: { + if (endmatch_ && p != end) + goto Next; + + // We found a match. If the caller doesn't care + // where the match is, no point going further. + if (nsubmatch_ == 0) + return true; + + // Record best match so far. + // Only need to check end point, because this entire + // call is only considering one start position. + matched = true; + cap_[1] = p; + if (submatch_[0].data() == NULL || + (longest_ && p > submatch_[0].end())) { + for (int i = 0; i < nsubmatch_; i++) + submatch_[i] = + StringPiece(cap_[2 * i], + static_cast(cap_[2 * i + 1] - cap_[2 * i])); + } + + // If going for first match, we're done. + if (!longest_) + return true; + + // If we used the entire text, no longer match is possible. + if (p == end) + return true; + + // Otherwise, continue on in hope of a longer match. + // Note the absence of the ShouldVisit() check here + // due to execution remaining in the same list. + Next: + if (!ip->last()) { + id++; + goto Loop; + } + break; + } + } + } + return matched; +} + +// Search text (within context) for prog_. +bool BitState::Search(const StringPiece& text, const StringPiece& context, + bool anchored, bool longest, + StringPiece* submatch, int nsubmatch) { + // Search parameters. + text_ = text; + context_ = context; + if (context_.begin() == NULL) + context_ = text; + if (prog_->anchor_start() && context_.begin() != text.begin()) + return false; + if (prog_->anchor_end() && context_.end() != text.end()) + return false; + anchored_ = anchored || prog_->anchor_start(); + longest_ = longest || prog_->anchor_end(); + endmatch_ = prog_->anchor_end(); + submatch_ = submatch; + nsubmatch_ = nsubmatch; + for (int i = 0; i < nsubmatch_; i++) + submatch_[i] = StringPiece(); + + // Allocate scratch space. + int nvisited = prog_->list_count() * static_cast(text.size()+1); + nvisited = (nvisited + VisitedBits-1) / VisitedBits; + visited_ = PODArray(nvisited); + memset(visited_.data(), 0, nvisited*sizeof visited_[0]); + + int ncap = 2*nsubmatch; + if (ncap < 2) + ncap = 2; + cap_ = PODArray(ncap); + memset(cap_.data(), 0, ncap*sizeof cap_[0]); + + // When sizeof(Job) == 16, we start with a nice round 1KiB. :) + job_ = PODArray(64); + + // Anchored search must start at text.begin(). + if (anchored_) { + cap_[0] = text.begin(); + return TrySearch(prog_->start(), text.begin()); + } + + // Unanchored search, starting from each possible text position. + // Notice that we have to try the empty string at the end of + // the text, so the loop condition is p <= text.end(), not p < text.end(). + // This looks like it's quadratic in the size of the text, + // but we are not clearing visited_ between calls to TrySearch, + // so no work is duplicated and it ends up still being linear. + for (const char* p = text.begin(); p <= text.end(); p++) { + // Try to use memchr to find the first byte quickly. + int fb = prog_->first_byte(); + if (fb >= 0 && p < text.end() && (p[0] & 0xFF) != fb) { + p = reinterpret_cast(memchr(p, fb, text.end() - p)); + if (p == NULL) + p = text.end(); + } + + cap_[0] = p; + if (TrySearch(prog_->start(), p)) // Match must be leftmost; done. + return true; + } + return false; +} + +// Bit-state search. +bool Prog::SearchBitState(const StringPiece& text, + const StringPiece& context, + Anchor anchor, + MatchKind kind, + StringPiece* match, + int nmatch) { + // If full match, we ask for an anchored longest match + // and then check that match[0] == text. + // So make sure match[0] exists. + StringPiece sp0; + if (kind == kFullMatch) { + anchor = kAnchored; + if (nmatch < 1) { + match = &sp0; + nmatch = 1; + } + } + + // Run the search. + BitState b(this); + bool anchored = anchor == kAnchored; + bool longest = kind != kFirstMatch; + if (!b.Search(text, context, anchored, longest, match, nmatch)) + return false; + if (kind == kFullMatch && match[0].end() != text.end()) + return false; + return true; +} + +} // namespace re2 diff --git a/extern/re2/re2/compile.cc b/extern/re2/re2/compile.cc new file mode 100644 index 0000000000..7457b228ac --- /dev/null +++ b/extern/re2/re2/compile.cc @@ -0,0 +1,1279 @@ +// Copyright 2007 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Compile regular expression to Prog. +// +// Prog and Inst are defined in prog.h. +// This file's external interface is just Regexp::CompileToProg. +// The Compiler class defined in this file is private. + +#include +#include +#include +#include + +#include "util/logging.h" +#include "util/pod_array.h" +#include "util/utf.h" +#include "re2/prog.h" +#include "re2/re2.h" +#include "re2/regexp.h" +#include "re2/walker-inl.h" + +namespace re2 { + +// List of pointers to Inst* that need to be filled in (patched). +// Because the Inst* haven't been filled in yet, +// we can use the Inst* word to hold the list's "next" pointer. +// It's kind of sleazy, but it works well in practice. +// See http://swtch.com/~rsc/regexp/regexp1.html for inspiration. +// +// Because the out and out1 fields in Inst are no longer pointers, +// we can't use pointers directly here either. Instead, p refers +// to inst_[p>>1].out (p&1 == 0) or inst_[p>>1].out1 (p&1 == 1). +// p == 0 represents the NULL list. This is okay because instruction #0 +// is always the fail instruction, which never appears on a list. + +struct PatchList { + uint32_t p; + + // Returns patch list containing just p. + static PatchList Mk(uint32_t p); + + // Patches all the entries on l to have value v. + // Caller must not ever use patch list again. + static void Patch(Prog::Inst *inst0, PatchList l, uint32_t v); + + // Deref returns the next pointer pointed at by p. + static PatchList Deref(Prog::Inst *inst0, PatchList l); + + // Appends two patch lists and returns result. + static PatchList Append(Prog::Inst *inst0, PatchList l1, PatchList l2); +}; + +static PatchList nullPatchList = { 0 }; + +// Returns patch list containing just p. +PatchList PatchList::Mk(uint32_t p) { + PatchList l; + l.p = p; + return l; +} + +// Returns the next pointer pointed at by l. +PatchList PatchList::Deref(Prog::Inst* inst0, PatchList l) { + Prog::Inst* ip = &inst0[l.p>>1]; + if (l.p&1) + l.p = ip->out1(); + else + l.p = ip->out(); + return l; +} + +// Patches all the entries on l to have value v. +void PatchList::Patch(Prog::Inst *inst0, PatchList l, uint32_t val) { + while (l.p != 0) { + Prog::Inst* ip = &inst0[l.p>>1]; + if (l.p&1) { + l.p = ip->out1(); + ip->out1_ = val; + } else { + l.p = ip->out(); + ip->set_out(val); + } + } +} + +// Appends two patch lists and returns result. +PatchList PatchList::Append(Prog::Inst* inst0, PatchList l1, PatchList l2) { + if (l1.p == 0) + return l2; + if (l2.p == 0) + return l1; + + PatchList l = l1; + for (;;) { + PatchList next = PatchList::Deref(inst0, l); + if (next.p == 0) + break; + l = next; + } + + Prog::Inst* ip = &inst0[l.p>>1]; + if (l.p&1) + ip->out1_ = l2.p; + else + ip->set_out(l2.p); + + return l1; +} + +// Compiled program fragment. +struct Frag { + uint32_t begin; + PatchList end; + + Frag() : begin(0) { end.p = 0; } // needed so Frag can go in vector + Frag(uint32_t begin, PatchList end) : begin(begin), end(end) {} +}; + +// Input encodings. +enum Encoding { + kEncodingUTF8 = 1, // UTF-8 (0-10FFFF) + kEncodingLatin1, // Latin-1 (0-FF) +}; + +class Compiler : public Regexp::Walker { + public: + explicit Compiler(); + ~Compiler(); + + // Compiles Regexp to a new Prog. + // Caller is responsible for deleting Prog when finished with it. + // If reversed is true, compiles for walking over the input + // string backward (reverses all concatenations). + static Prog *Compile(Regexp* re, bool reversed, int64_t max_mem); + + // Compiles alternation of all the re to a new Prog. + // Each re has a match with an id equal to its index in the vector. + static Prog* CompileSet(Regexp* re, RE2::Anchor anchor, int64_t max_mem); + + // Interface for Regexp::Walker, which helps traverse the Regexp. + // The walk is purely post-recursive: given the machines for the + // children, PostVisit combines them to create the machine for + // the current node. The child_args are Frags. + // The Compiler traverses the Regexp parse tree, visiting + // each node in depth-first order. It invokes PreVisit before + // visiting the node's children and PostVisit after visiting + // the children. + Frag PreVisit(Regexp* re, Frag parent_arg, bool* stop); + Frag PostVisit(Regexp* re, Frag parent_arg, Frag pre_arg, Frag* child_args, + int nchild_args); + Frag ShortVisit(Regexp* re, Frag parent_arg); + Frag Copy(Frag arg); + + // Given fragment a, returns a+ or a+?; a* or a*?; a? or a?? + Frag Plus(Frag a, bool nongreedy); + Frag Star(Frag a, bool nongreedy); + Frag Quest(Frag a, bool nongreedy); + + // Given fragment a, returns (a) capturing as \n. + Frag Capture(Frag a, int n); + + // Given fragments a and b, returns ab; a|b + Frag Cat(Frag a, Frag b); + Frag Alt(Frag a, Frag b); + + // Returns a fragment that can't match anything. + Frag NoMatch(); + + // Returns a fragment that matches the empty string. + Frag Match(int32_t id); + + // Returns a no-op fragment. + Frag Nop(); + + // Returns a fragment matching the byte range lo-hi. + Frag ByteRange(int lo, int hi, bool foldcase); + + // Returns a fragment matching an empty-width special op. + Frag EmptyWidth(EmptyOp op); + + // Adds n instructions to the program. + // Returns the index of the first one. + // Returns -1 if no more instructions are available. + int AllocInst(int n); + + // Rune range compiler. + + // Begins a new alternation. + void BeginRange(); + + // Adds a fragment matching the rune range lo-hi. + void AddRuneRange(Rune lo, Rune hi, bool foldcase); + void AddRuneRangeLatin1(Rune lo, Rune hi, bool foldcase); + void AddRuneRangeUTF8(Rune lo, Rune hi, bool foldcase); + void Add_80_10ffff(); + + // New suffix that matches the byte range lo-hi, then goes to next. + int UncachedRuneByteSuffix(uint8_t lo, uint8_t hi, bool foldcase, int next); + int CachedRuneByteSuffix(uint8_t lo, uint8_t hi, bool foldcase, int next); + + // Returns true iff the suffix is cached. + bool IsCachedRuneByteSuffix(int id); + + // Adds a suffix to alternation. + void AddSuffix(int id); + + // Adds a suffix to the trie starting from the given root node. + // Returns zero iff allocating an instruction fails. Otherwise, returns + // the current root node, which might be different from what was given. + int AddSuffixRecursive(int root, int id); + + // Finds the trie node for the given suffix. Returns a Frag in order to + // distinguish between pointing at the root node directly (end.p == 0) + // and pointing at an Alt's out1 or out (end.p&1 == 1 or 0, respectively). + Frag FindByteRange(int root, int id); + + // Compares two ByteRanges and returns true iff they are equal. + bool ByteRangeEqual(int id1, int id2); + + // Returns the alternation of all the added suffixes. + Frag EndRange(); + + // Single rune. + Frag Literal(Rune r, bool foldcase); + + void Setup(Regexp::ParseFlags, int64_t, RE2::Anchor); + Prog* Finish(); + + // Returns .* where dot = any byte + Frag DotStar(); + + private: + Prog* prog_; // Program being built. + bool failed_; // Did we give up compiling? + Encoding encoding_; // Input encoding + bool reversed_; // Should program run backward over text? + + PODArray inst_; + int ninst_; // Number of instructions used. + int max_ninst_; // Maximum number of instructions. + + int64_t max_mem_; // Total memory budget. + + std::unordered_map rune_cache_; + Frag rune_range_; + + RE2::Anchor anchor_; // anchor mode for RE2::Set + + Compiler(const Compiler&) = delete; + Compiler& operator=(const Compiler&) = delete; +}; + +Compiler::Compiler() { + prog_ = new Prog(); + failed_ = false; + encoding_ = kEncodingUTF8; + reversed_ = false; + ninst_ = 0; + max_ninst_ = 1; // make AllocInst for fail instruction okay + max_mem_ = 0; + int fail = AllocInst(1); + inst_[fail].InitFail(); + max_ninst_ = 0; // Caller must change +} + +Compiler::~Compiler() { + delete prog_; +} + +int Compiler::AllocInst(int n) { + if (failed_ || ninst_ + n > max_ninst_) { + failed_ = true; + return -1; + } + + if (ninst_ + n > inst_.size()) { + int cap = inst_.size(); + if (cap == 0) + cap = 8; + while (ninst_ + n > cap) + cap *= 2; + PODArray inst(cap); + if (inst_.data() != NULL) + memmove(inst.data(), inst_.data(), ninst_*sizeof inst_[0]); + memset(inst.data() + ninst_, 0, (cap - ninst_)*sizeof inst_[0]); + inst_ = std::move(inst); + } + int id = ninst_; + ninst_ += n; + return id; +} + +// These routines are somewhat hard to visualize in text -- +// see http://swtch.com/~rsc/regexp/regexp1.html for +// pictures explaining what is going on here. + +// Returns an unmatchable fragment. +Frag Compiler::NoMatch() { + return Frag(0, nullPatchList); +} + +// Is a an unmatchable fragment? +static bool IsNoMatch(Frag a) { + return a.begin == 0; +} + +// Given fragments a and b, returns fragment for ab. +Frag Compiler::Cat(Frag a, Frag b) { + if (IsNoMatch(a) || IsNoMatch(b)) + return NoMatch(); + + // Elide no-op. + Prog::Inst* begin = &inst_[a.begin]; + if (begin->opcode() == kInstNop && + a.end.p == (a.begin << 1) && + begin->out() == 0) { + // in case refs to a somewhere + PatchList::Patch(inst_.data(), a.end, b.begin); + return b; + } + + // To run backward over string, reverse all concatenations. + if (reversed_) { + PatchList::Patch(inst_.data(), b.end, a.begin); + return Frag(b.begin, a.end); + } + + PatchList::Patch(inst_.data(), a.end, b.begin); + return Frag(a.begin, b.end); +} + +// Given fragments for a and b, returns fragment for a|b. +Frag Compiler::Alt(Frag a, Frag b) { + // Special case for convenience in loops. + if (IsNoMatch(a)) + return b; + if (IsNoMatch(b)) + return a; + + int id = AllocInst(1); + if (id < 0) + return NoMatch(); + + inst_[id].InitAlt(a.begin, b.begin); + return Frag(id, PatchList::Append(inst_.data(), a.end, b.end)); +} + +// When capturing submatches in like-Perl mode, a kOpAlt Inst +// treats out_ as the first choice, out1_ as the second. +// +// For *, +, and ?, if out_ causes another repetition, +// then the operator is greedy. If out1_ is the repetition +// (and out_ moves forward), then the operator is non-greedy. + +// Given a fragment a, returns a fragment for a* or a*? (if nongreedy) +Frag Compiler::Star(Frag a, bool nongreedy) { + int id = AllocInst(1); + if (id < 0) + return NoMatch(); + inst_[id].InitAlt(0, 0); + PatchList::Patch(inst_.data(), a.end, id); + if (nongreedy) { + inst_[id].out1_ = a.begin; + return Frag(id, PatchList::Mk(id << 1)); + } else { + inst_[id].set_out(a.begin); + return Frag(id, PatchList::Mk((id << 1) | 1)); + } +} + +// Given a fragment for a, returns a fragment for a+ or a+? (if nongreedy) +Frag Compiler::Plus(Frag a, bool nongreedy) { + // a+ is just a* with a different entry point. + Frag f = Star(a, nongreedy); + return Frag(a.begin, f.end); +} + +// Given a fragment for a, returns a fragment for a? or a?? (if nongreedy) +Frag Compiler::Quest(Frag a, bool nongreedy) { + if (IsNoMatch(a)) + return Nop(); + int id = AllocInst(1); + if (id < 0) + return NoMatch(); + PatchList pl; + if (nongreedy) { + inst_[id].InitAlt(0, a.begin); + pl = PatchList::Mk(id << 1); + } else { + inst_[id].InitAlt(a.begin, 0); + pl = PatchList::Mk((id << 1) | 1); + } + return Frag(id, PatchList::Append(inst_.data(), pl, a.end)); +} + +// Returns a fragment for the byte range lo-hi. +Frag Compiler::ByteRange(int lo, int hi, bool foldcase) { + int id = AllocInst(1); + if (id < 0) + return NoMatch(); + inst_[id].InitByteRange(lo, hi, foldcase, 0); + return Frag(id, PatchList::Mk(id << 1)); +} + +// Returns a no-op fragment. Sometimes unavoidable. +Frag Compiler::Nop() { + int id = AllocInst(1); + if (id < 0) + return NoMatch(); + inst_[id].InitNop(0); + return Frag(id, PatchList::Mk(id << 1)); +} + +// Returns a fragment that signals a match. +Frag Compiler::Match(int32_t match_id) { + int id = AllocInst(1); + if (id < 0) + return NoMatch(); + inst_[id].InitMatch(match_id); + return Frag(id, nullPatchList); +} + +// Returns a fragment matching a particular empty-width op (like ^ or $) +Frag Compiler::EmptyWidth(EmptyOp empty) { + int id = AllocInst(1); + if (id < 0) + return NoMatch(); + inst_[id].InitEmptyWidth(empty, 0); + return Frag(id, PatchList::Mk(id << 1)); +} + +// Given a fragment a, returns a fragment with capturing parens around a. +Frag Compiler::Capture(Frag a, int n) { + if (IsNoMatch(a)) + return NoMatch(); + int id = AllocInst(2); + if (id < 0) + return NoMatch(); + inst_[id].InitCapture(2*n, a.begin); + inst_[id+1].InitCapture(2*n+1, 0); + PatchList::Patch(inst_.data(), a.end, id+1); + + return Frag(id, PatchList::Mk((id+1) << 1)); +} + +// A Rune is a name for a Unicode code point. +// Returns maximum rune encoded by UTF-8 sequence of length len. +static int MaxRune(int len) { + int b; // number of Rune bits in len-byte UTF-8 sequence (len < UTFmax) + if (len == 1) + b = 7; + else + b = 8-(len+1) + 6*(len-1); + return (1<::const_iterator it = rune_cache_.find(key); + if (it != rune_cache_.end()) + return it->second; + int id = UncachedRuneByteSuffix(lo, hi, foldcase, next); + rune_cache_[key] = id; + return id; +} + +bool Compiler::IsCachedRuneByteSuffix(int id) { + uint8_t lo = inst_[id].lo_; + uint8_t hi = inst_[id].hi_; + bool foldcase = inst_[id].foldcase() != 0; + int next = inst_[id].out(); + + uint64_t key = MakeRuneCacheKey(lo, hi, foldcase, next); + return rune_cache_.find(key) != rune_cache_.end(); +} + +void Compiler::AddSuffix(int id) { + if (failed_) + return; + + if (rune_range_.begin == 0) { + rune_range_.begin = id; + return; + } + + if (encoding_ == kEncodingUTF8) { + // Build a trie in order to reduce fanout. + rune_range_.begin = AddSuffixRecursive(rune_range_.begin, id); + return; + } + + int alt = AllocInst(1); + if (alt < 0) { + rune_range_.begin = 0; + return; + } + inst_[alt].InitAlt(rune_range_.begin, id); + rune_range_.begin = alt; +} + +int Compiler::AddSuffixRecursive(int root, int id) { + DCHECK(inst_[root].opcode() == kInstAlt || + inst_[root].opcode() == kInstByteRange); + + Frag f = FindByteRange(root, id); + if (IsNoMatch(f)) { + int alt = AllocInst(1); + if (alt < 0) + return 0; + inst_[alt].InitAlt(root, id); + return alt; + } + + int br; + if (f.end.p == 0) + br = root; + else if (f.end.p&1) + br = inst_[f.begin].out1(); + else + br = inst_[f.begin].out(); + + if (IsCachedRuneByteSuffix(br)) { + // We can't fiddle with cached suffixes, so make a clone of the head. + int byterange = AllocInst(1); + if (byterange < 0) + return 0; + inst_[byterange].InitByteRange(inst_[br].lo(), inst_[br].hi(), + inst_[br].foldcase(), inst_[br].out()); + + // Ensure that the parent points to the clone, not to the original. + // Note that this could leave the head unreachable except via the cache. + br = byterange; + if (f.end.p == 0) + root = br; + else if (f.end.p&1) + inst_[f.begin].out1_ = br; + else + inst_[f.begin].set_out(br); + } + + int out = inst_[id].out(); + if (!IsCachedRuneByteSuffix(id)) { + // The head should be the instruction most recently allocated, so free it + // instead of leaving it unreachable. + DCHECK_EQ(id, ninst_-1); + inst_[id].out_opcode_ = 0; + inst_[id].out1_ = 0; + ninst_--; + } + + out = AddSuffixRecursive(inst_[br].out(), out); + if (out == 0) + return 0; + + inst_[br].set_out(out); + return root; +} + +bool Compiler::ByteRangeEqual(int id1, int id2) { + return inst_[id1].lo() == inst_[id2].lo() && + inst_[id1].hi() == inst_[id2].hi() && + inst_[id1].foldcase() == inst_[id2].foldcase(); +} + +Frag Compiler::FindByteRange(int root, int id) { + if (inst_[root].opcode() == kInstByteRange) { + if (ByteRangeEqual(root, id)) + return Frag(root, nullPatchList); + else + return NoMatch(); + } + + while (inst_[root].opcode() == kInstAlt) { + int out1 = inst_[root].out1(); + if (ByteRangeEqual(out1, id)) + return Frag(root, PatchList::Mk((root << 1) | 1)); + + // CharClass is a sorted list of ranges, so if out1 of the root Alt wasn't + // what we're looking for, then we can stop immediately. Unfortunately, we + // can't short-circuit the search in reverse mode. + if (!reversed_) + return NoMatch(); + + int out = inst_[root].out(); + if (inst_[out].opcode() == kInstAlt) + root = out; + else if (ByteRangeEqual(out, id)) + return Frag(root, PatchList::Mk(root << 1)); + else + return NoMatch(); + } + + LOG(DFATAL) << "should never happen"; + return NoMatch(); +} + +Frag Compiler::EndRange() { + return rune_range_; +} + +// Converts rune range lo-hi into a fragment that recognizes +// the bytes that would make up those runes in the current +// encoding (Latin 1 or UTF-8). +// This lets the machine work byte-by-byte even when +// using multibyte encodings. + +void Compiler::AddRuneRange(Rune lo, Rune hi, bool foldcase) { + switch (encoding_) { + default: + case kEncodingUTF8: + AddRuneRangeUTF8(lo, hi, foldcase); + break; + case kEncodingLatin1: + AddRuneRangeLatin1(lo, hi, foldcase); + break; + } +} + +void Compiler::AddRuneRangeLatin1(Rune lo, Rune hi, bool foldcase) { + // Latin-1 is easy: runes *are* bytes. + if (lo > hi || lo > 0xFF) + return; + if (hi > 0xFF) + hi = 0xFF; + AddSuffix(UncachedRuneByteSuffix(static_cast(lo), + static_cast(hi), foldcase, 0)); +} + +// Table describing how to make a UTF-8 matching machine +// for the rune range 80-10FFFF (Runeself-Runemax). +// This range happens frequently enough (for example /./ and /[^a-z]/) +// and the rune_cache_ map is slow enough that this is worth +// special handling. Makes compilation of a small expression +// with a dot in it about 10% faster. +// The * in the comments below mark whole sequences. +static struct ByteRangeProg { + int next; + int lo; + int hi; +} prog_80_10ffff[] = { + // Two-byte + { -1, 0x80, 0xBF, }, // 0: 80-BF + { 0, 0xC2, 0xDF, }, // 1: C2-DF 80-BF* + + // Three-byte + { 0, 0xA0, 0xBF, }, // 2: A0-BF 80-BF + { 2, 0xE0, 0xE0, }, // 3: E0 A0-BF 80-BF* + { 0, 0x80, 0xBF, }, // 4: 80-BF 80-BF + { 4, 0xE1, 0xEF, }, // 5: E1-EF 80-BF 80-BF* + + // Four-byte + { 4, 0x90, 0xBF, }, // 6: 90-BF 80-BF 80-BF + { 6, 0xF0, 0xF0, }, // 7: F0 90-BF 80-BF 80-BF* + { 4, 0x80, 0xBF, }, // 8: 80-BF 80-BF 80-BF + { 8, 0xF1, 0xF3, }, // 9: F1-F3 80-BF 80-BF 80-BF* + { 4, 0x80, 0x8F, }, // 10: 80-8F 80-BF 80-BF + { 10, 0xF4, 0xF4, }, // 11: F4 80-8F 80-BF 80-BF* +}; + +void Compiler::Add_80_10ffff() { + int inst[arraysize(prog_80_10ffff)] = { 0 }; // does not need to be initialized; silences gcc warning + for (size_t i = 0; i < arraysize(prog_80_10ffff); i++) { + const ByteRangeProg& p = prog_80_10ffff[i]; + int next = 0; + if (p.next >= 0) + next = inst[p.next]; + inst[i] = UncachedRuneByteSuffix(static_cast(p.lo), + static_cast(p.hi), false, next); + if ((p.lo & 0xC0) != 0x80) + AddSuffix(inst[i]); + } +} + +void Compiler::AddRuneRangeUTF8(Rune lo, Rune hi, bool foldcase) { + if (lo > hi) + return; + + // Pick off 80-10FFFF as a common special case + // that can bypass the slow rune_cache_. + if (lo == 0x80 && hi == 0x10ffff && !reversed_) { + Add_80_10ffff(); + return; + } + + // Split range into same-length sized ranges. + for (int i = 1; i < UTFmax; i++) { + Rune max = MaxRune(i); + if (lo <= max && max < hi) { + AddRuneRangeUTF8(lo, max, foldcase); + AddRuneRangeUTF8(max+1, hi, foldcase); + return; + } + } + + // ASCII range is always a special case. + if (hi < Runeself) { + AddSuffix(UncachedRuneByteSuffix(static_cast(lo), + static_cast(hi), foldcase, 0)); + return; + } + + // Split range into sections that agree on leading bytes. + for (int i = 1; i < UTFmax; i++) { + uint32_t m = (1<<(6*i)) - 1; // last i bytes of a UTF-8 sequence + if ((lo & ~m) != (hi & ~m)) { + if ((lo & m) != 0) { + AddRuneRangeUTF8(lo, lo|m, foldcase); + AddRuneRangeUTF8((lo|m)+1, hi, foldcase); + return; + } + if ((hi & m) != m) { + AddRuneRangeUTF8(lo, (hi&~m)-1, foldcase); + AddRuneRangeUTF8(hi&~m, hi, foldcase); + return; + } + } + } + + // Finally. Generate byte matching equivalent for lo-hi. + uint8_t ulo[UTFmax], uhi[UTFmax]; + int n = runetochar(reinterpret_cast(ulo), &lo); + int m = runetochar(reinterpret_cast(uhi), &hi); + (void)m; // USED(m) + DCHECK_EQ(n, m); + + // The logic below encodes this thinking: + // + // 1. When we have built the whole suffix, we know that it cannot + // possibly be a suffix of anything longer: in forward mode, nothing + // else can occur before the leading byte; in reverse mode, nothing + // else can occur after the last continuation byte or else the leading + // byte would have to change. Thus, there is no benefit to caching + // the first byte of the suffix whereas there is a cost involved in + // cloning it if it begins a common prefix, which is fairly likely. + // + // 2. Conversely, the last byte of the suffix cannot possibly be a + // prefix of anything because next == 0, so we will never want to + // clone it, but it is fairly likely to be a common suffix. Perhaps + // more so in reverse mode than in forward mode because the former is + // "converging" towards lower entropy, but caching is still worthwhile + // for the latter in cases such as 80-BF. + // + // 3. Handling the bytes between the first and the last is less + // straightforward and, again, the approach depends on whether we are + // "converging" towards lower entropy: in forward mode, a single byte + // is unlikely to be part of a common suffix whereas a byte range + // is more likely so; in reverse mode, a byte range is unlikely to + // be part of a common suffix whereas a single byte is more likely + // so. The same benefit versus cost argument applies here. + int id = 0; + if (reversed_) { + for (int i = 0; i < n; i++) { + // In reverse UTF-8 mode: cache the leading byte; don't cache the last + // continuation byte; cache anything else iff it's a single byte (XX-XX). + if (i == 0 || (ulo[i] == uhi[i] && i != n-1)) + id = CachedRuneByteSuffix(ulo[i], uhi[i], false, id); + else + id = UncachedRuneByteSuffix(ulo[i], uhi[i], false, id); + } + } else { + for (int i = n-1; i >= 0; i--) { + // In forward UTF-8 mode: don't cache the leading byte; cache the last + // continuation byte; cache anything else iff it's a byte range (XX-YY). + if (i == n-1 || (ulo[i] < uhi[i] && i != 0)) + id = CachedRuneByteSuffix(ulo[i], uhi[i], false, id); + else + id = UncachedRuneByteSuffix(ulo[i], uhi[i], false, id); + } + } + AddSuffix(id); +} + +// Should not be called. +Frag Compiler::Copy(Frag arg) { + // We're using WalkExponential; there should be no copying. + LOG(DFATAL) << "Compiler::Copy called!"; + failed_ = true; + return NoMatch(); +} + +// Visits a node quickly; called once WalkExponential has +// decided to cut this walk short. +Frag Compiler::ShortVisit(Regexp* re, Frag) { + failed_ = true; + return NoMatch(); +} + +// Called before traversing a node's children during the walk. +Frag Compiler::PreVisit(Regexp* re, Frag, bool* stop) { + // Cut off walk if we've already failed. + if (failed_) + *stop = true; + + return Frag(); // not used by caller +} + +Frag Compiler::Literal(Rune r, bool foldcase) { + switch (encoding_) { + default: + return Frag(); + + case kEncodingLatin1: + return ByteRange(r, r, foldcase); + + case kEncodingUTF8: { + if (r < Runeself) // Make common case fast. + return ByteRange(r, r, foldcase); + uint8_t buf[UTFmax]; + int n = runetochar(reinterpret_cast(buf), &r); + Frag f = ByteRange((uint8_t)buf[0], buf[0], false); + for (int i = 1; i < n; i++) + f = Cat(f, ByteRange((uint8_t)buf[i], buf[i], false)); + return f; + } + } +} + +// Called after traversing the node's children during the walk. +// Given their frags, build and return the frag for this re. +Frag Compiler::PostVisit(Regexp* re, Frag, Frag, Frag* child_frags, + int nchild_frags) { + // If a child failed, don't bother going forward, especially + // since the child_frags might contain Frags with NULLs in them. + if (failed_) + return NoMatch(); + + // Given the child fragments, return the fragment for this node. + switch (re->op()) { + case kRegexpRepeat: + // Should not see; code at bottom of function will print error + break; + + case kRegexpNoMatch: + return NoMatch(); + + case kRegexpEmptyMatch: + return Nop(); + + case kRegexpHaveMatch: { + Frag f = Match(re->match_id()); + if (anchor_ == RE2::ANCHOR_BOTH) { + // Append \z or else the subexpression will effectively be unanchored. + // Complemented by the UNANCHORED case in CompileSet(). + f = Cat(EmptyWidth(kEmptyEndText), f); + } + return f; + } + + case kRegexpConcat: { + Frag f = child_frags[0]; + for (int i = 1; i < nchild_frags; i++) + f = Cat(f, child_frags[i]); + return f; + } + + case kRegexpAlternate: { + Frag f = child_frags[0]; + for (int i = 1; i < nchild_frags; i++) + f = Alt(f, child_frags[i]); + return f; + } + + case kRegexpStar: + return Star(child_frags[0], (re->parse_flags()&Regexp::NonGreedy) != 0); + + case kRegexpPlus: + return Plus(child_frags[0], (re->parse_flags()&Regexp::NonGreedy) != 0); + + case kRegexpQuest: + return Quest(child_frags[0], (re->parse_flags()&Regexp::NonGreedy) != 0); + + case kRegexpLiteral: + return Literal(re->rune(), (re->parse_flags()&Regexp::FoldCase) != 0); + + case kRegexpLiteralString: { + // Concatenation of literals. + if (re->nrunes() == 0) + return Nop(); + Frag f; + for (int i = 0; i < re->nrunes(); i++) { + Frag f1 = Literal(re->runes()[i], + (re->parse_flags()&Regexp::FoldCase) != 0); + if (i == 0) + f = f1; + else + f = Cat(f, f1); + } + return f; + } + + case kRegexpAnyChar: + BeginRange(); + AddRuneRange(0, Runemax, false); + return EndRange(); + + case kRegexpAnyByte: + return ByteRange(0x00, 0xFF, false); + + case kRegexpCharClass: { + CharClass* cc = re->cc(); + if (cc->empty()) { + // This can't happen. + LOG(DFATAL) << "No ranges in char class"; + failed_ = true; + return NoMatch(); + } + + // ASCII case-folding optimization: if the char class + // behaves the same on A-Z as it does on a-z, + // discard any ranges wholly contained in A-Z + // and mark the other ranges as foldascii. + // This reduces the size of a program for + // (?i)abc from 3 insts per letter to 1 per letter. + bool foldascii = cc->FoldsASCII(); + + // Character class is just a big OR of the different + // character ranges in the class. + BeginRange(); + for (CharClass::iterator i = cc->begin(); i != cc->end(); ++i) { + // ASCII case-folding optimization (see above). + if (foldascii && 'A' <= i->lo && i->hi <= 'Z') + continue; + + // If this range contains all of A-Za-z or none of it, + // the fold flag is unnecessary; don't bother. + bool fold = foldascii; + if ((i->lo <= 'A' && 'z' <= i->hi) || i->hi < 'A' || 'z' < i->lo || + ('Z' < i->lo && i->hi < 'a')) + fold = false; + + AddRuneRange(i->lo, i->hi, fold); + } + return EndRange(); + } + + case kRegexpCapture: + // If this is a non-capturing parenthesis -- (?:foo) -- + // just use the inner expression. + if (re->cap() < 0) + return child_frags[0]; + return Capture(child_frags[0], re->cap()); + + case kRegexpBeginLine: + return EmptyWidth(reversed_ ? kEmptyEndLine : kEmptyBeginLine); + + case kRegexpEndLine: + return EmptyWidth(reversed_ ? kEmptyBeginLine : kEmptyEndLine); + + case kRegexpBeginText: + return EmptyWidth(reversed_ ? kEmptyEndText : kEmptyBeginText); + + case kRegexpEndText: + return EmptyWidth(reversed_ ? kEmptyBeginText : kEmptyEndText); + + case kRegexpWordBoundary: + return EmptyWidth(kEmptyWordBoundary); + + case kRegexpNoWordBoundary: + return EmptyWidth(kEmptyNonWordBoundary); + } + LOG(DFATAL) << "Missing case in Compiler: " << re->op(); + failed_ = true; + return NoMatch(); +} + +// Is this regexp required to start at the beginning of the text? +// Only approximate; can return false for complicated regexps like (\Aa|\Ab), +// but handles (\A(a|b)). Could use the Walker to write a more exact one. +static bool IsAnchorStart(Regexp** pre, int depth) { + Regexp* re = *pre; + Regexp* sub; + // The depth limit makes sure that we don't overflow + // the stack on a deeply nested regexp. As the comment + // above says, IsAnchorStart is conservative, so returning + // a false negative is okay. The exact limit is somewhat arbitrary. + if (re == NULL || depth >= 4) + return false; + switch (re->op()) { + default: + break; + case kRegexpConcat: + if (re->nsub() > 0) { + sub = re->sub()[0]->Incref(); + if (IsAnchorStart(&sub, depth+1)) { + PODArray subcopy(re->nsub()); + subcopy[0] = sub; // already have reference + for (int i = 1; i < re->nsub(); i++) + subcopy[i] = re->sub()[i]->Incref(); + *pre = Regexp::Concat(subcopy.data(), re->nsub(), re->parse_flags()); + re->Decref(); + return true; + } + sub->Decref(); + } + break; + case kRegexpCapture: + sub = re->sub()[0]->Incref(); + if (IsAnchorStart(&sub, depth+1)) { + *pre = Regexp::Capture(sub, re->parse_flags(), re->cap()); + re->Decref(); + return true; + } + sub->Decref(); + break; + case kRegexpBeginText: + *pre = Regexp::LiteralString(NULL, 0, re->parse_flags()); + re->Decref(); + return true; + } + return false; +} + +// Is this regexp required to start at the end of the text? +// Only approximate; can return false for complicated regexps like (a\z|b\z), +// but handles ((a|b)\z). Could use the Walker to write a more exact one. +static bool IsAnchorEnd(Regexp** pre, int depth) { + Regexp* re = *pre; + Regexp* sub; + // The depth limit makes sure that we don't overflow + // the stack on a deeply nested regexp. As the comment + // above says, IsAnchorEnd is conservative, so returning + // a false negative is okay. The exact limit is somewhat arbitrary. + if (re == NULL || depth >= 4) + return false; + switch (re->op()) { + default: + break; + case kRegexpConcat: + if (re->nsub() > 0) { + sub = re->sub()[re->nsub() - 1]->Incref(); + if (IsAnchorEnd(&sub, depth+1)) { + PODArray subcopy(re->nsub()); + subcopy[re->nsub() - 1] = sub; // already have reference + for (int i = 0; i < re->nsub() - 1; i++) + subcopy[i] = re->sub()[i]->Incref(); + *pre = Regexp::Concat(subcopy.data(), re->nsub(), re->parse_flags()); + re->Decref(); + return true; + } + sub->Decref(); + } + break; + case kRegexpCapture: + sub = re->sub()[0]->Incref(); + if (IsAnchorEnd(&sub, depth+1)) { + *pre = Regexp::Capture(sub, re->parse_flags(), re->cap()); + re->Decref(); + return true; + } + sub->Decref(); + break; + case kRegexpEndText: + *pre = Regexp::LiteralString(NULL, 0, re->parse_flags()); + re->Decref(); + return true; + } + return false; +} + +void Compiler::Setup(Regexp::ParseFlags flags, int64_t max_mem, + RE2::Anchor anchor) { + prog_->set_flags(flags); + + if (flags & Regexp::Latin1) + encoding_ = kEncodingLatin1; + max_mem_ = max_mem; + if (max_mem <= 0) { + max_ninst_ = 100000; // more than enough + } else if (static_cast(max_mem) <= sizeof(Prog)) { + // No room for anything. + max_ninst_ = 0; + } else { + int64_t m = (max_mem - sizeof(Prog)) / sizeof(Prog::Inst); + // Limit instruction count so that inst->id() fits nicely in an int. + // SparseArray also assumes that the indices (inst->id()) are ints. + // The call to WalkExponential uses 2*max_ninst_ below, + // and other places in the code use 2 or 3 * prog->size(). + // Limiting to 2^24 should avoid overflow in those places. + // (The point of allowing more than 32 bits of memory is to + // have plenty of room for the DFA states, not to use it up + // on the program.) + if (m >= 1<<24) + m = 1<<24; + + // Inst imposes its own limit (currently bigger than 2^24 but be safe). + if (m > Prog::Inst::kMaxInst) + m = Prog::Inst::kMaxInst; + + max_ninst_ = static_cast(m); + } + + anchor_ = anchor; +} + +// Compiles re, returning program. +// Caller is responsible for deleting prog_. +// If reversed is true, compiles a program that expects +// to run over the input string backward (reverses all concatenations). +// The reversed flag is also recorded in the returned program. +Prog* Compiler::Compile(Regexp* re, bool reversed, int64_t max_mem) { + Compiler c; + c.Setup(re->parse_flags(), max_mem, RE2::UNANCHORED /* unused */); + c.reversed_ = reversed; + + // Simplify to remove things like counted repetitions + // and character classes like \d. + Regexp* sre = re->Simplify(); + if (sre == NULL) + return NULL; + + // Record whether prog is anchored, removing the anchors. + // (They get in the way of other optimizations.) + bool is_anchor_start = IsAnchorStart(&sre, 0); + bool is_anchor_end = IsAnchorEnd(&sre, 0); + + // Generate fragment for entire regexp. + Frag all = c.WalkExponential(sre, Frag(), 2*c.max_ninst_); + sre->Decref(); + if (c.failed_) + return NULL; + + // Success! Finish by putting Match node at end, and record start. + // Turn off c.reversed_ (if it is set) to force the remaining concatenations + // to behave normally. + c.reversed_ = false; + all = c.Cat(all, c.Match(0)); + + c.prog_->set_reversed(reversed); + if (c.prog_->reversed()) { + c.prog_->set_anchor_start(is_anchor_end); + c.prog_->set_anchor_end(is_anchor_start); + } else { + c.prog_->set_anchor_start(is_anchor_start); + c.prog_->set_anchor_end(is_anchor_end); + } + + c.prog_->set_start(all.begin); + if (!c.prog_->anchor_start()) { + // Also create unanchored version, which starts with a .*? loop. + all = c.Cat(c.DotStar(), all); + } + c.prog_->set_start_unanchored(all.begin); + + // Hand ownership of prog_ to caller. + return c.Finish(); +} + +Prog* Compiler::Finish() { + if (failed_) + return NULL; + + if (prog_->start() == 0 && prog_->start_unanchored() == 0) { + // No possible matches; keep Fail instruction only. + ninst_ = 1; + } + + // Hand off the array to Prog. + prog_->inst_ = std::move(inst_); + prog_->size_ = ninst_; + + prog_->Optimize(); + prog_->Flatten(); + prog_->ComputeByteMap(); + + // Record remaining memory for DFA. + if (max_mem_ <= 0) { + prog_->set_dfa_mem(1<<20); + } else { + int64_t m = max_mem_ - sizeof(Prog); + m -= prog_->size_*sizeof(Prog::Inst); // account for inst_ + if (prog_->CanBitState()) + m -= prog_->size_*sizeof(uint16_t); // account for list_heads_ + if (m < 0) + m = 0; + prog_->set_dfa_mem(m); + } + + Prog* p = prog_; + prog_ = NULL; + return p; +} + +// Converts Regexp to Prog. +Prog* Regexp::CompileToProg(int64_t max_mem) { + return Compiler::Compile(this, false, max_mem); +} + +Prog* Regexp::CompileToReverseProg(int64_t max_mem) { + return Compiler::Compile(this, true, max_mem); +} + +Frag Compiler::DotStar() { + return Star(ByteRange(0x00, 0xff, false), true); +} + +// Compiles RE set to Prog. +Prog* Compiler::CompileSet(Regexp* re, RE2::Anchor anchor, int64_t max_mem) { + Compiler c; + c.Setup(re->parse_flags(), max_mem, anchor); + + Regexp* sre = re->Simplify(); + if (sre == NULL) + return NULL; + + Frag all = c.WalkExponential(sre, Frag(), 2*c.max_ninst_); + sre->Decref(); + if (c.failed_) + return NULL; + + c.prog_->set_anchor_start(true); + c.prog_->set_anchor_end(true); + + if (anchor == RE2::UNANCHORED) { + // Prepend .* or else the expression will effectively be anchored. + // Complemented by the ANCHOR_BOTH case in PostVisit(). + all = c.Cat(c.DotStar(), all); + } + c.prog_->set_start(all.begin); + c.prog_->set_start_unanchored(all.begin); + + Prog* prog = c.Finish(); + if (prog == NULL) + return NULL; + + // Make sure DFA has enough memory to operate, + // since we're not going to fall back to the NFA. + bool dfa_failed = false; + StringPiece sp = "hello, world"; + prog->SearchDFA(sp, sp, Prog::kAnchored, Prog::kManyMatch, + NULL, &dfa_failed, NULL); + if (dfa_failed) { + delete prog; + return NULL; + } + + return prog; +} + +Prog* Prog::CompileSet(Regexp* re, RE2::Anchor anchor, int64_t max_mem) { + return Compiler::CompileSet(re, anchor, max_mem); +} + +} // namespace re2 diff --git a/extern/re2/re2/dfa.cc b/extern/re2/re2/dfa.cc new file mode 100644 index 0000000000..40880f9e2b --- /dev/null +++ b/extern/re2/re2/dfa.cc @@ -0,0 +1,2151 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// A DFA (deterministic finite automaton)-based regular expression search. +// +// The DFA search has two main parts: the construction of the automaton, +// which is represented by a graph of State structures, and the execution +// of the automaton over a given input string. +// +// The basic idea is that the State graph is constructed so that the +// execution can simply start with a state s, and then for each byte c in +// the input string, execute "s = s->next[c]", checking at each point whether +// the current s represents a matching state. +// +// The simple explanation just given does convey the essence of this code, +// but it omits the details of how the State graph gets constructed as well +// as some performance-driven optimizations to the execution of the automaton. +// All these details are explained in the comments for the code following +// the definition of class DFA. +// +// See http://swtch.com/~rsc/regexp/ for a very bare-bones equivalent. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util/logging.h" +#include "util/mix.h" +#include "util/mutex.h" +#include "util/pod_array.h" +#include "util/sparse_set.h" +#include "util/strutil.h" +#include "re2/prog.h" +#include "re2/stringpiece.h" + +// Silence "zero-sized array in struct/union" warning for DFA::State::next_. +#ifdef _MSC_VER +#pragma warning(disable: 4200) +#endif + +namespace re2 { + +#if !defined(__linux__) /* only Linux seems to have memrchr */ +static void* memrchr(const void* s, int c, size_t n) { + const unsigned char* p = (const unsigned char*)s; + for (p += n; n > 0; n--) + if (*--p == c) + return (void*)p; + + return NULL; +} +#endif + +// Controls whether the DFA should bail out early if the NFA would be faster. +static bool dfa_should_bail_when_slow = true; + +// Changing this to true compiles in prints that trace execution of the DFA. +// Generates a lot of output -- only useful for debugging. +static const bool ExtraDebug = false; + +// A DFA implementation of a regular expression program. +// Since this is entirely a forward declaration mandated by C++, +// some of the comments here are better understood after reading +// the comments in the sections that follow the DFA definition. +class DFA { + public: + DFA(Prog* prog, Prog::MatchKind kind, int64_t max_mem); + ~DFA(); + bool ok() const { return !init_failed_; } + Prog::MatchKind kind() { return kind_; } + + // Searches for the regular expression in text, which is considered + // as a subsection of context for the purposes of interpreting flags + // like ^ and $ and \A and \z. + // Returns whether a match was found. + // If a match is found, sets *ep to the end point of the best match in text. + // If "anchored", the match must begin at the start of text. + // If "want_earliest_match", the match that ends first is used, not + // necessarily the best one. + // If "run_forward" is true, the DFA runs from text.begin() to text.end(). + // If it is false, the DFA runs from text.end() to text.begin(), + // returning the leftmost end of the match instead of the rightmost one. + // If the DFA cannot complete the search (for example, if it is out of + // memory), it sets *failed and returns false. + bool Search(const StringPiece& text, const StringPiece& context, + bool anchored, bool want_earliest_match, bool run_forward, + bool* failed, const char** ep, SparseSet* matches); + + // Builds out all states for the entire DFA. + // If cb is not empty, it receives one callback per state built. + // Returns the number of states built. + // FOR TESTING OR EXPERIMENTAL PURPOSES ONLY. + int BuildAllStates(const Prog::DFAStateCallback& cb); + + // Computes min and max for matching strings. Won't return strings + // bigger than maxlen. + bool PossibleMatchRange(std::string* min, std::string* max, int maxlen); + + // These data structures are logically private, but C++ makes it too + // difficult to mark them as such. + class RWLocker; + class StateSaver; + class Workq; + + // A single DFA state. The DFA is represented as a graph of these + // States, linked by the next_ pointers. If in state s and reading + // byte c, the next state should be s->next_[c]. + struct State { + inline bool IsMatch() const { return (flag_ & kFlagMatch) != 0; } + + int* inst_; // Instruction pointers in the state. + int ninst_; // # of inst_ pointers. + uint32_t flag_; // Empty string bitfield flags in effect on the way + // into this state, along with kFlagMatch if this + // is a matching state. + +// Work around the bug affecting flexible array members in GCC 6.x (for x >= 1). +// (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70932) +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ == 6 && __GNUC_MINOR__ >= 1 + std::atomic next_[0]; // Outgoing arrows from State, +#else + std::atomic next_[]; // Outgoing arrows from State, +#endif + + // one per input byte class + }; + + enum { + kByteEndText = 256, // imaginary byte at end of text + + kFlagEmptyMask = 0xFF, // State.flag_: bits holding kEmptyXXX flags + kFlagMatch = 0x0100, // State.flag_: this is a matching state + kFlagLastWord = 0x0200, // State.flag_: last byte was a word char + kFlagNeedShift = 16, // needed kEmpty bits are or'ed in shifted left + }; + + struct StateHash { + size_t operator()(const State* a) const { + DCHECK(a != NULL); + HashMix mix(a->flag_); + for (int i = 0; i < a->ninst_; i++) + mix.Mix(a->inst_[i]); + mix.Mix(0); + return mix.get(); + } + }; + + struct StateEqual { + bool operator()(const State* a, const State* b) const { + DCHECK(a != NULL); + DCHECK(b != NULL); + if (a == b) + return true; + if (a->flag_ != b->flag_) + return false; + if (a->ninst_ != b->ninst_) + return false; + for (int i = 0; i < a->ninst_; i++) + if (a->inst_[i] != b->inst_[i]) + return false; + return true; + } + }; + + typedef std::unordered_set StateSet; + + private: + // Special "first_byte" values for a state. (Values >= 0 denote actual bytes.) + enum { + kFbUnknown = -1, // No analysis has been performed. + kFbNone = -2, // The first-byte trick cannot be used. + }; + + enum { + // Indices into start_ for unanchored searches. + // Add kStartAnchored for anchored searches. + kStartBeginText = 0, // text at beginning of context + kStartBeginLine = 2, // text at beginning of line + kStartAfterWordChar = 4, // text follows a word character + kStartAfterNonWordChar = 6, // text follows non-word character + kMaxStart = 8, + + kStartAnchored = 1, + }; + + // Resets the DFA State cache, flushing all saved State* information. + // Releases and reacquires cache_mutex_ via cache_lock, so any + // State* existing before the call are not valid after the call. + // Use a StateSaver to preserve important states across the call. + // cache_mutex_.r <= L < mutex_ + // After: cache_mutex_.w <= L < mutex_ + void ResetCache(RWLocker* cache_lock); + + // Looks up and returns the State corresponding to a Workq. + // L >= mutex_ + State* WorkqToCachedState(Workq* q, Workq* mq, uint32_t flag); + + // Looks up and returns a State matching the inst, ninst, and flag. + // L >= mutex_ + State* CachedState(int* inst, int ninst, uint32_t flag); + + // Clear the cache entirely. + // Must hold cache_mutex_.w or be in destructor. + void ClearCache(); + + // Converts a State into a Workq: the opposite of WorkqToCachedState. + // L >= mutex_ + void StateToWorkq(State* s, Workq* q); + + // Runs a State on a given byte, returning the next state. + State* RunStateOnByteUnlocked(State*, int); // cache_mutex_.r <= L < mutex_ + State* RunStateOnByte(State*, int); // L >= mutex_ + + // Runs a Workq on a given byte followed by a set of empty-string flags, + // producing a new Workq in nq. If a match instruction is encountered, + // sets *ismatch to true. + // L >= mutex_ + void RunWorkqOnByte(Workq* q, Workq* nq, + int c, uint32_t flag, bool* ismatch); + + // Runs a Workq on a set of empty-string flags, producing a new Workq in nq. + // L >= mutex_ + void RunWorkqOnEmptyString(Workq* q, Workq* nq, uint32_t flag); + + // Adds the instruction id to the Workq, following empty arrows + // according to flag. + // L >= mutex_ + void AddToQueue(Workq* q, int id, uint32_t flag); + + // For debugging, returns a text representation of State. + static std::string DumpState(State* state); + + // For debugging, returns a text representation of a Workq. + static std::string DumpWorkq(Workq* q); + + // Search parameters + struct SearchParams { + SearchParams(const StringPiece& text, const StringPiece& context, + RWLocker* cache_lock) + : text(text), context(context), + anchored(false), + want_earliest_match(false), + run_forward(false), + start(NULL), + first_byte(kFbUnknown), + cache_lock(cache_lock), + failed(false), + ep(NULL), + matches(NULL) { } + + StringPiece text; + StringPiece context; + bool anchored; + bool want_earliest_match; + bool run_forward; + State* start; + int first_byte; + RWLocker *cache_lock; + bool failed; // "out" parameter: whether search gave up + const char* ep; // "out" parameter: end pointer for match + SparseSet* matches; + + private: + SearchParams(const SearchParams&) = delete; + SearchParams& operator=(const SearchParams&) = delete; + }; + + // Before each search, the parameters to Search are analyzed by + // AnalyzeSearch to determine the state in which to start and the + // "first_byte" for that state, if any. + struct StartInfo { + StartInfo() : start(NULL), first_byte(kFbUnknown) {} + State* start; + std::atomic first_byte; + }; + + // Fills in params->start and params->first_byte using + // the other search parameters. Returns true on success, + // false on failure. + // cache_mutex_.r <= L < mutex_ + bool AnalyzeSearch(SearchParams* params); + bool AnalyzeSearchHelper(SearchParams* params, StartInfo* info, + uint32_t flags); + + // The generic search loop, inlined to create specialized versions. + // cache_mutex_.r <= L < mutex_ + // Might unlock and relock cache_mutex_ via params->cache_lock. + inline bool InlinedSearchLoop(SearchParams* params, + bool have_first_byte, + bool want_earliest_match, + bool run_forward); + + // The specialized versions of InlinedSearchLoop. The three letters + // at the ends of the name denote the true/false values used as the + // last three parameters of InlinedSearchLoop. + // cache_mutex_.r <= L < mutex_ + // Might unlock and relock cache_mutex_ via params->cache_lock. + bool SearchFFF(SearchParams* params); + bool SearchFFT(SearchParams* params); + bool SearchFTF(SearchParams* params); + bool SearchFTT(SearchParams* params); + bool SearchTFF(SearchParams* params); + bool SearchTFT(SearchParams* params); + bool SearchTTF(SearchParams* params); + bool SearchTTT(SearchParams* params); + + // The main search loop: calls an appropriate specialized version of + // InlinedSearchLoop. + // cache_mutex_.r <= L < mutex_ + // Might unlock and relock cache_mutex_ via params->cache_lock. + bool FastSearchLoop(SearchParams* params); + + // For debugging, a slow search loop that calls InlinedSearchLoop + // directly -- because the booleans passed are not constants, the + // loop is not specialized like the SearchFFF etc. versions, so it + // runs much more slowly. Useful only for debugging. + // cache_mutex_.r <= L < mutex_ + // Might unlock and relock cache_mutex_ via params->cache_lock. + bool SlowSearchLoop(SearchParams* params); + + // Looks up bytes in bytemap_ but handles case c == kByteEndText too. + int ByteMap(int c) { + if (c == kByteEndText) + return prog_->bytemap_range(); + return prog_->bytemap()[c]; + } + + // Constant after initialization. + Prog* prog_; // The regular expression program to run. + Prog::MatchKind kind_; // The kind of DFA. + bool init_failed_; // initialization failed (out of memory) + + Mutex mutex_; // mutex_ >= cache_mutex_.r + + // Scratch areas, protected by mutex_. + Workq* q0_; // Two pre-allocated work queues. + Workq* q1_; + PODArray stack_; // Pre-allocated stack for AddToQueue + + // State* cache. Many threads use and add to the cache simultaneously, + // holding cache_mutex_ for reading and mutex_ (above) when adding. + // If the cache fills and needs to be discarded, the discarding is done + // while holding cache_mutex_ for writing, to avoid interrupting other + // readers. Any State* pointers are only valid while cache_mutex_ + // is held. + Mutex cache_mutex_; + int64_t mem_budget_; // Total memory budget for all States. + int64_t state_budget_; // Amount of memory remaining for new States. + StateSet state_cache_; // All States computed so far. + StartInfo start_[kMaxStart]; +}; + +// Shorthand for casting to uint8_t*. +static inline const uint8_t* BytePtr(const void* v) { + return reinterpret_cast(v); +} + +// Work queues + +// Marks separate thread groups of different priority +// in the work queue when in leftmost-longest matching mode. +#define Mark (-1) + +// Separates the match IDs from the instructions in inst_. +// Used only for "many match" DFA states. +#define MatchSep (-2) + +// Internally, the DFA uses a sparse array of +// program instruction pointers as a work queue. +// In leftmost longest mode, marks separate sections +// of workq that started executing at different +// locations in the string (earlier locations first). +class DFA::Workq : public SparseSet { + public: + // Constructor: n is number of normal slots, maxmark number of mark slots. + Workq(int n, int maxmark) : + SparseSet(n+maxmark), + n_(n), + maxmark_(maxmark), + nextmark_(n), + last_was_mark_(true) { + } + + bool is_mark(int i) { return i >= n_; } + + int maxmark() { return maxmark_; } + + void clear() { + SparseSet::clear(); + nextmark_ = n_; + } + + void mark() { + if (last_was_mark_) + return; + last_was_mark_ = false; + SparseSet::insert_new(nextmark_++); + } + + int size() { + return n_ + maxmark_; + } + + void insert(int id) { + if (contains(id)) + return; + insert_new(id); + } + + void insert_new(int id) { + last_was_mark_ = false; + SparseSet::insert_new(id); + } + + private: + int n_; // size excluding marks + int maxmark_; // maximum number of marks + int nextmark_; // id of next mark + bool last_was_mark_; // last inserted was mark + + Workq(const Workq&) = delete; + Workq& operator=(const Workq&) = delete; +}; + +DFA::DFA(Prog* prog, Prog::MatchKind kind, int64_t max_mem) + : prog_(prog), + kind_(kind), + init_failed_(false), + q0_(NULL), + q1_(NULL), + mem_budget_(max_mem) { + if (ExtraDebug) + fprintf(stderr, "\nkind %d\n%s\n", (int)kind_, prog_->DumpUnanchored().c_str()); + int nmark = 0; + if (kind_ == Prog::kLongestMatch) + nmark = prog_->size(); + // See DFA::AddToQueue() for why this is so. + int nstack = prog_->inst_count(kInstCapture) + + prog_->inst_count(kInstEmptyWidth) + + prog_->inst_count(kInstNop) + + nmark + 1; // + 1 for start inst + + // Account for space needed for DFA, q0, q1, stack. + mem_budget_ -= sizeof(DFA); + mem_budget_ -= (prog_->size() + nmark) * + (sizeof(int)+sizeof(int)) * 2; // q0, q1 + mem_budget_ -= nstack * sizeof(int); // stack + if (mem_budget_ < 0) { + init_failed_ = true; + return; + } + + state_budget_ = mem_budget_; + + // Make sure there is a reasonable amount of working room left. + // At minimum, the search requires room for two states in order + // to limp along, restarting frequently. We'll get better performance + // if there is room for a larger number of states, say 20. + // Note that a state stores list heads only, so we use the program + // list count for the upper bound, not the program size. + int nnext = prog_->bytemap_range() + 1; // + 1 for kByteEndText slot + int64_t one_state = sizeof(State) + nnext*sizeof(std::atomic) + + (prog_->list_count()+nmark)*sizeof(int); + if (state_budget_ < 20*one_state) { + init_failed_ = true; + return; + } + + q0_ = new Workq(prog_->size(), nmark); + q1_ = new Workq(prog_->size(), nmark); + stack_ = PODArray(nstack); +} + +DFA::~DFA() { + delete q0_; + delete q1_; + ClearCache(); +} + +// In the DFA state graph, s->next[c] == NULL means that the +// state has not yet been computed and needs to be. We need +// a different special value to signal that s->next[c] is a +// state that can never lead to a match (and thus the search +// can be called off). Hence DeadState. +#define DeadState reinterpret_cast(1) + +// Signals that the rest of the string matches no matter what it is. +#define FullMatchState reinterpret_cast(2) + +#define SpecialStateMax FullMatchState + +// Debugging printouts + +// For debugging, returns a string representation of the work queue. +std::string DFA::DumpWorkq(Workq* q) { + std::string s; + const char* sep = ""; + for (Workq::iterator it = q->begin(); it != q->end(); ++it) { + if (q->is_mark(*it)) { + s += "|"; + sep = ""; + } else { + s += StringPrintf("%s%d", sep, *it); + sep = ","; + } + } + return s; +} + +// For debugging, returns a string representation of the state. +std::string DFA::DumpState(State* state) { + if (state == NULL) + return "_"; + if (state == DeadState) + return "X"; + if (state == FullMatchState) + return "*"; + std::string s; + const char* sep = ""; + s += StringPrintf("(%p)", state); + for (int i = 0; i < state->ninst_; i++) { + if (state->inst_[i] == Mark) { + s += "|"; + sep = ""; + } else if (state->inst_[i] == MatchSep) { + s += "||"; + sep = ""; + } else { + s += StringPrintf("%s%d", sep, state->inst_[i]); + sep = ","; + } + } + s += StringPrintf(" flag=%#x", state->flag_); + return s; +} + +////////////////////////////////////////////////////////////////////// +// +// DFA state graph construction. +// +// The DFA state graph is a heavily-linked collection of State* structures. +// The state_cache_ is a set of all the State structures ever allocated, +// so that if the same state is reached by two different paths, +// the same State structure can be used. This reduces allocation +// requirements and also avoids duplication of effort across the two +// identical states. +// +// A State is defined by an ordered list of instruction ids and a flag word. +// +// The choice of an ordered list of instructions differs from a typical +// textbook DFA implementation, which would use an unordered set. +// Textbook descriptions, however, only care about whether +// the DFA matches, not where it matches in the text. To decide where the +// DFA matches, we need to mimic the behavior of the dominant backtracking +// implementations like PCRE, which try one possible regular expression +// execution, then another, then another, stopping when one of them succeeds. +// The DFA execution tries these many executions in parallel, representing +// each by an instruction id. These pointers are ordered in the State.inst_ +// list in the same order that the executions would happen in a backtracking +// search: if a match is found during execution of inst_[2], inst_[i] for i>=3 +// can be discarded. +// +// Textbooks also typically do not consider context-aware empty string operators +// like ^ or $. These are handled by the flag word, which specifies the set +// of empty-string operators that should be matched when executing at the +// current text position. These flag bits are defined in prog.h. +// The flag word also contains two DFA-specific bits: kFlagMatch if the state +// is a matching state (one that reached a kInstMatch in the program) +// and kFlagLastWord if the last processed byte was a word character, for the +// implementation of \B and \b. +// +// The flag word also contains, shifted up 16 bits, the bits looked for by +// any kInstEmptyWidth instructions in the state. These provide a useful +// summary indicating when new flags might be useful. +// +// The permanent representation of a State's instruction ids is just an array, +// but while a state is being analyzed, these instruction ids are represented +// as a Workq, which is an array that allows iteration in insertion order. + +// NOTE(rsc): The choice of State construction determines whether the DFA +// mimics backtracking implementations (so-called leftmost first matching) or +// traditional DFA implementations (so-called leftmost longest matching as +// prescribed by POSIX). This implementation chooses to mimic the +// backtracking implementations, because we want to replace PCRE. To get +// POSIX behavior, the states would need to be considered not as a simple +// ordered list of instruction ids, but as a list of unordered sets of instruction +// ids. A match by a state in one set would inhibit the running of sets +// farther down the list but not other instruction ids in the same set. Each +// set would correspond to matches beginning at a given point in the string. +// This is implemented by separating different sets with Mark pointers. + +// Looks in the State cache for a State matching q, flag. +// If one is found, returns it. If one is not found, allocates one, +// inserts it in the cache, and returns it. +// If mq is not null, MatchSep and the match IDs in mq will be appended +// to the State. +DFA::State* DFA::WorkqToCachedState(Workq* q, Workq* mq, uint32_t flag) { + //mutex_.AssertHeld(); + + // Construct array of instruction ids for the new state. + // Only ByteRange, EmptyWidth, and Match instructions are useful to keep: + // those are the only operators with any effect in + // RunWorkqOnEmptyString or RunWorkqOnByte. + int* inst = new int[q->size()]; + int n = 0; + uint32_t needflags = 0; // flags needed by kInstEmptyWidth instructions + bool sawmatch = false; // whether queue contains guaranteed kInstMatch + bool sawmark = false; // whether queue contains a Mark + if (ExtraDebug) + fprintf(stderr, "WorkqToCachedState %s [%#x]", DumpWorkq(q).c_str(), flag); + for (Workq::iterator it = q->begin(); it != q->end(); ++it) { + int id = *it; + if (sawmatch && (kind_ == Prog::kFirstMatch || q->is_mark(id))) + break; + if (q->is_mark(id)) { + if (n > 0 && inst[n-1] != Mark) { + sawmark = true; + inst[n++] = Mark; + } + continue; + } + Prog::Inst* ip = prog_->inst(id); + switch (ip->opcode()) { + case kInstAltMatch: + // This state will continue to a match no matter what + // the rest of the input is. If it is the highest priority match + // being considered, return the special FullMatchState + // to indicate that it's all matches from here out. + if (kind_ != Prog::kManyMatch && + (kind_ != Prog::kFirstMatch || + (it == q->begin() && ip->greedy(prog_))) && + (kind_ != Prog::kLongestMatch || !sawmark) && + (flag & kFlagMatch)) { + delete[] inst; + if (ExtraDebug) + fprintf(stderr, " -> FullMatchState\n"); + return FullMatchState; + } + FALLTHROUGH_INTENDED; + default: + // Record iff id is the head of its list, which must + // be the case if id-1 is the last of *its* list. :) + if (prog_->inst(id-1)->last()) + inst[n++] = *it; + if (ip->opcode() == kInstEmptyWidth) + needflags |= ip->empty(); + if (ip->opcode() == kInstMatch && !prog_->anchor_end()) + sawmatch = true; + break; + } + } + DCHECK_LE(n, q->size()); + if (n > 0 && inst[n-1] == Mark) + n--; + + // If there are no empty-width instructions waiting to execute, + // then the extra flag bits will not be used, so there is no + // point in saving them. (Discarding them reduces the number + // of distinct states.) + if (needflags == 0) + flag &= kFlagMatch; + + // NOTE(rsc): The code above cannot do flag &= needflags, + // because if the right flags were present to pass the current + // kInstEmptyWidth instructions, new kInstEmptyWidth instructions + // might be reached that in turn need different flags. + // The only sure thing is that if there are no kInstEmptyWidth + // instructions at all, no flags will be needed. + // We could do the extra work to figure out the full set of + // possibly needed flags by exploring past the kInstEmptyWidth + // instructions, but the check above -- are any flags needed + // at all? -- handles the most common case. More fine-grained + // analysis can only be justified by measurements showing that + // too many redundant states are being allocated. + + // If there are no Insts in the list, it's a dead state, + // which is useful to signal with a special pointer so that + // the execution loop can stop early. This is only okay + // if the state is *not* a matching state. + if (n == 0 && flag == 0) { + delete[] inst; + if (ExtraDebug) + fprintf(stderr, " -> DeadState\n"); + return DeadState; + } + + // If we're in longest match mode, the state is a sequence of + // unordered state sets separated by Marks. Sort each set + // to canonicalize, to reduce the number of distinct sets stored. + if (kind_ == Prog::kLongestMatch) { + int* ip = inst; + int* ep = ip + n; + while (ip < ep) { + int* markp = ip; + while (markp < ep && *markp != Mark) + markp++; + std::sort(ip, markp); + if (markp < ep) + markp++; + ip = markp; + } + } + + // If we're in many match mode, canonicalize for similar reasons: + // we have an unordered set of states (i.e. we don't have Marks) + // and sorting will reduce the number of distinct sets stored. + if (kind_ == Prog::kManyMatch) { + int* ip = inst; + int* ep = ip + n; + std::sort(ip, ep); + } + + // Append MatchSep and the match IDs in mq if necessary. + if (mq != NULL) { + inst[n++] = MatchSep; + for (Workq::iterator i = mq->begin(); i != mq->end(); ++i) { + int id = *i; + Prog::Inst* ip = prog_->inst(id); + if (ip->opcode() == kInstMatch) + inst[n++] = ip->match_id(); + } + } + + // Save the needed empty-width flags in the top bits for use later. + flag |= needflags << kFlagNeedShift; + + State* state = CachedState(inst, n, flag); + delete[] inst; + return state; +} + +// Looks in the State cache for a State matching inst, ninst, flag. +// If one is found, returns it. If one is not found, allocates one, +// inserts it in the cache, and returns it. +DFA::State* DFA::CachedState(int* inst, int ninst, uint32_t flag) { + //mutex_.AssertHeld(); + + // Look in the cache for a pre-existing state. + // We have to initialise the struct like this because otherwise + // MSVC will complain about the flexible array member. :( + State state; + state.inst_ = inst; + state.ninst_ = ninst; + state.flag_ = flag; + StateSet::iterator it = state_cache_.find(&state); + if (it != state_cache_.end()) { + if (ExtraDebug) + fprintf(stderr, " -cached-> %s\n", DumpState(*it).c_str()); + return *it; + } + + // Must have enough memory for new state. + // In addition to what we're going to allocate, + // the state cache hash table seems to incur about 40 bytes per + // State*, empirically. + const int kStateCacheOverhead = 40; + int nnext = prog_->bytemap_range() + 1; // + 1 for kByteEndText slot + int mem = sizeof(State) + nnext*sizeof(std::atomic) + + ninst*sizeof(int); + if (mem_budget_ < mem + kStateCacheOverhead) { + mem_budget_ = -1; + return NULL; + } + mem_budget_ -= mem + kStateCacheOverhead; + + // Allocate new state along with room for next_ and inst_. + char* space = std::allocator().allocate(mem); + State* s = new (space) State; + (void) new (s->next_) std::atomic[nnext]; + // Work around a unfortunate bug in older versions of libstdc++. + // (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64658) + for (int i = 0; i < nnext; i++) + (void) new (s->next_ + i) std::atomic(NULL); + s->inst_ = new (s->next_ + nnext) int[ninst]; + memmove(s->inst_, inst, ninst*sizeof s->inst_[0]); + s->ninst_ = ninst; + s->flag_ = flag; + if (ExtraDebug) + fprintf(stderr, " -> %s\n", DumpState(s).c_str()); + + // Put state in cache and return it. + state_cache_.insert(s); + return s; +} + +// Clear the cache. Must hold cache_mutex_.w or be in destructor. +void DFA::ClearCache() { + StateSet::iterator begin = state_cache_.begin(); + StateSet::iterator end = state_cache_.end(); + while (begin != end) { + StateSet::iterator tmp = begin; + ++begin; + // Deallocate the blob of memory that we allocated in DFA::CachedState(). + // We recompute mem in order to benefit from sized delete where possible. + int ninst = (*tmp)->ninst_; + int nnext = prog_->bytemap_range() + 1; // + 1 for kByteEndText slot + int mem = sizeof(State) + nnext*sizeof(std::atomic) + + ninst*sizeof(int); + std::allocator().deallocate(reinterpret_cast(*tmp), mem); + } + state_cache_.clear(); +} + +// Copies insts in state s to the work queue q. +void DFA::StateToWorkq(State* s, Workq* q) { + q->clear(); + for (int i = 0; i < s->ninst_; i++) { + if (s->inst_[i] == Mark) { + q->mark(); + } else if (s->inst_[i] == MatchSep) { + // Nothing after this is an instruction! + break; + } else { + // Explore from the head of the list. + AddToQueue(q, s->inst_[i], s->flag_ & kFlagEmptyMask); + } + } +} + +// Adds ip to the work queue, following empty arrows according to flag. +void DFA::AddToQueue(Workq* q, int id, uint32_t flag) { + + // Use stack_ to hold our stack of instructions yet to process. + // It was preallocated as follows: + // one entry per Capture; + // one entry per EmptyWidth; and + // one entry per Nop. + // This reflects the maximum number of stack pushes that each can + // perform. (Each instruction can be processed at most once.) + // When using marks, we also added nmark == prog_->size(). + // (Otherwise, nmark == 0.) + int* stk = stack_.data(); + int nstk = 0; + + stk[nstk++] = id; + while (nstk > 0) { + DCHECK_LE(nstk, stack_.size()); + id = stk[--nstk]; + + Loop: + if (id == Mark) { + q->mark(); + continue; + } + + if (id == 0) + continue; + + // If ip is already on the queue, nothing to do. + // Otherwise add it. We don't actually keep all the + // ones that get added, but adding all of them here + // increases the likelihood of q->contains(id), + // reducing the amount of duplicated work. + if (q->contains(id)) + continue; + q->insert_new(id); + + // Process instruction. + Prog::Inst* ip = prog_->inst(id); + switch (ip->opcode()) { + default: + LOG(DFATAL) << "unhandled opcode: " << ip->opcode(); + break; + + case kInstByteRange: // just save these on the queue + case kInstMatch: + if (ip->last()) + break; + id = id+1; + goto Loop; + + case kInstCapture: // DFA treats captures as no-ops. + case kInstNop: + if (!ip->last()) + stk[nstk++] = id+1; + + // If this instruction is the [00-FF]* loop at the beginning of + // a leftmost-longest unanchored search, separate with a Mark so + // that future threads (which will start farther to the right in + // the input string) are lower priority than current threads. + if (ip->opcode() == kInstNop && q->maxmark() > 0 && + id == prog_->start_unanchored() && id != prog_->start()) + stk[nstk++] = Mark; + id = ip->out(); + goto Loop; + + case kInstAltMatch: + DCHECK(!ip->last()); + id = id+1; + goto Loop; + + case kInstEmptyWidth: + if (!ip->last()) + stk[nstk++] = id+1; + + // Continue on if we have all the right flag bits. + if (ip->empty() & ~flag) + break; + id = ip->out(); + goto Loop; + } + } +} + +// Running of work queues. In the work queue, order matters: +// the queue is sorted in priority order. If instruction i comes before j, +// then the instructions that i produces during the run must come before +// the ones that j produces. In order to keep this invariant, all the +// work queue runners have to take an old queue to process and then +// also a new queue to fill in. It's not acceptable to add to the end of +// an existing queue, because new instructions will not end up in the +// correct position. + +// Runs the work queue, processing the empty strings indicated by flag. +// For example, flag == kEmptyBeginLine|kEmptyEndLine means to match +// both ^ and $. It is important that callers pass all flags at once: +// processing both ^ and $ is not the same as first processing only ^ +// and then processing only $. Doing the two-step sequence won't match +// ^$^$^$ but processing ^ and $ simultaneously will (and is the behavior +// exhibited by existing implementations). +void DFA::RunWorkqOnEmptyString(Workq* oldq, Workq* newq, uint32_t flag) { + newq->clear(); + for (Workq::iterator i = oldq->begin(); i != oldq->end(); ++i) { + if (oldq->is_mark(*i)) + AddToQueue(newq, Mark, flag); + else + AddToQueue(newq, *i, flag); + } +} + +// Runs the work queue, processing the single byte c followed by any empty +// strings indicated by flag. For example, c == 'a' and flag == kEmptyEndLine, +// means to match c$. Sets the bool *ismatch to true if the end of the +// regular expression program has been reached (the regexp has matched). +void DFA::RunWorkqOnByte(Workq* oldq, Workq* newq, + int c, uint32_t flag, bool* ismatch) { + //mutex_.AssertHeld(); + + newq->clear(); + for (Workq::iterator i = oldq->begin(); i != oldq->end(); ++i) { + if (oldq->is_mark(*i)) { + if (*ismatch) + return; + newq->mark(); + continue; + } + int id = *i; + Prog::Inst* ip = prog_->inst(id); + switch (ip->opcode()) { + default: + LOG(DFATAL) << "unhandled opcode: " << ip->opcode(); + break; + + case kInstFail: // never succeeds + case kInstCapture: // already followed + case kInstNop: // already followed + case kInstAltMatch: // already followed + case kInstEmptyWidth: // already followed + break; + + case kInstByteRange: // can follow if c is in range + if (ip->Matches(c)) + AddToQueue(newq, ip->out(), flag); + break; + + case kInstMatch: + if (prog_->anchor_end() && c != kByteEndText && + kind_ != Prog::kManyMatch) + break; + *ismatch = true; + if (kind_ == Prog::kFirstMatch) { + // Can stop processing work queue since we found a match. + return; + } + break; + } + } + + if (ExtraDebug) + fprintf(stderr, "%s on %d[%#x] -> %s [%d]\n", DumpWorkq(oldq).c_str(), + c, flag, DumpWorkq(newq).c_str(), *ismatch); +} + +// Processes input byte c in state, returning new state. +// Caller does not hold mutex. +DFA::State* DFA::RunStateOnByteUnlocked(State* state, int c) { + // Keep only one RunStateOnByte going + // even if the DFA is being run by multiple threads. + MutexLock l(&mutex_); + return RunStateOnByte(state, c); +} + +// Processes input byte c in state, returning new state. +DFA::State* DFA::RunStateOnByte(State* state, int c) { + //mutex_.AssertHeld(); + + if (state <= SpecialStateMax) { + if (state == FullMatchState) { + // It is convenient for routines like PossibleMatchRange + // if we implement RunStateOnByte for FullMatchState: + // once you get into this state you never get out, + // so it's pretty easy. + return FullMatchState; + } + if (state == DeadState) { + LOG(DFATAL) << "DeadState in RunStateOnByte"; + return NULL; + } + if (state == NULL) { + LOG(DFATAL) << "NULL state in RunStateOnByte"; + return NULL; + } + LOG(DFATAL) << "Unexpected special state in RunStateOnByte"; + return NULL; + } + + // If someone else already computed this, return it. + State* ns = state->next_[ByteMap(c)].load(std::memory_order_relaxed); + if (ns != NULL) + return ns; + + // Convert state into Workq. + StateToWorkq(state, q0_); + + // Flags marking the kinds of empty-width things (^ $ etc) + // around this byte. Before the byte we have the flags recorded + // in the State structure itself. After the byte we have + // nothing yet (but that will change: read on). + uint32_t needflag = state->flag_ >> kFlagNeedShift; + uint32_t beforeflag = state->flag_ & kFlagEmptyMask; + uint32_t oldbeforeflag = beforeflag; + uint32_t afterflag = 0; + + if (c == '\n') { + // Insert implicit $ and ^ around \n + beforeflag |= kEmptyEndLine; + afterflag |= kEmptyBeginLine; + } + + if (c == kByteEndText) { + // Insert implicit $ and \z before the fake "end text" byte. + beforeflag |= kEmptyEndLine | kEmptyEndText; + } + + // The state flag kFlagLastWord says whether the last + // byte processed was a word character. Use that info to + // insert empty-width (non-)word boundaries. + bool islastword = (state->flag_ & kFlagLastWord) != 0; + bool isword = c != kByteEndText && Prog::IsWordChar(static_cast(c)); + if (isword == islastword) + beforeflag |= kEmptyNonWordBoundary; + else + beforeflag |= kEmptyWordBoundary; + + // Okay, finally ready to run. + // Only useful to rerun on empty string if there are new, useful flags. + if (beforeflag & ~oldbeforeflag & needflag) { + RunWorkqOnEmptyString(q0_, q1_, beforeflag); + using std::swap; + swap(q0_, q1_); + } + bool ismatch = false; + RunWorkqOnByte(q0_, q1_, c, afterflag, &ismatch); + using std::swap; + swap(q0_, q1_); + + // Save afterflag along with ismatch and isword in new state. + uint32_t flag = afterflag; + if (ismatch) + flag |= kFlagMatch; + if (isword) + flag |= kFlagLastWord; + + if (ismatch && kind_ == Prog::kManyMatch) + ns = WorkqToCachedState(q0_, q1_, flag); + else + ns = WorkqToCachedState(q0_, NULL, flag); + + // Flush ns before linking to it. + // Write barrier before updating state->next_ so that the + // main search loop can proceed without any locking, for speed. + // (Otherwise it would need one mutex operation per input byte.) + state->next_[ByteMap(c)].store(ns, std::memory_order_release); + return ns; +} + + +////////////////////////////////////////////////////////////////////// +// DFA cache reset. + +// Reader-writer lock helper. +// +// The DFA uses a reader-writer mutex to protect the state graph itself. +// Traversing the state graph requires holding the mutex for reading, +// and discarding the state graph and starting over requires holding the +// lock for writing. If a search needs to expand the graph but is out +// of memory, it will need to drop its read lock and then acquire the +// write lock. Since it cannot then atomically downgrade from write lock +// to read lock, it runs the rest of the search holding the write lock. +// (This probably helps avoid repeated contention, but really the decision +// is forced by the Mutex interface.) It's a bit complicated to keep +// track of whether the lock is held for reading or writing and thread +// that through the search, so instead we encapsulate it in the RWLocker +// and pass that around. + +class DFA::RWLocker { + public: + explicit RWLocker(Mutex* mu); + ~RWLocker(); + + // If the lock is only held for reading right now, + // drop the read lock and re-acquire for writing. + // Subsequent calls to LockForWriting are no-ops. + // Notice that the lock is *released* temporarily. + void LockForWriting(); + + private: + Mutex* mu_; + bool writing_; + + RWLocker(const RWLocker&) = delete; + RWLocker& operator=(const RWLocker&) = delete; +}; + +DFA::RWLocker::RWLocker(Mutex* mu) : mu_(mu), writing_(false) { + mu_->ReaderLock(); +} + +// This function is marked as NO_THREAD_SAFETY_ANALYSIS because the annotations +// does not support lock upgrade. +void DFA::RWLocker::LockForWriting() NO_THREAD_SAFETY_ANALYSIS { + if (!writing_) { + mu_->ReaderUnlock(); + mu_->WriterLock(); + writing_ = true; + } +} + +DFA::RWLocker::~RWLocker() { + if (!writing_) + mu_->ReaderUnlock(); + else + mu_->WriterUnlock(); +} + + +// When the DFA's State cache fills, we discard all the states in the +// cache and start over. Many threads can be using and adding to the +// cache at the same time, so we synchronize using the cache_mutex_ +// to keep from stepping on other threads. Specifically, all the +// threads using the current cache hold cache_mutex_ for reading. +// When a thread decides to flush the cache, it drops cache_mutex_ +// and then re-acquires it for writing. That ensures there are no +// other threads accessing the cache anymore. The rest of the search +// runs holding cache_mutex_ for writing, avoiding any contention +// with or cache pollution caused by other threads. + +void DFA::ResetCache(RWLocker* cache_lock) { + // Re-acquire the cache_mutex_ for writing (exclusive use). + cache_lock->LockForWriting(); + + // Clear the cache, reset the memory budget. + for (int i = 0; i < kMaxStart; i++) { + start_[i].start = NULL; + start_[i].first_byte.store(kFbUnknown, std::memory_order_relaxed); + } + ClearCache(); + mem_budget_ = state_budget_; +} + +// Typically, a couple States do need to be preserved across a cache +// reset, like the State at the current point in the search. +// The StateSaver class helps keep States across cache resets. +// It makes a copy of the state's guts outside the cache (before the reset) +// and then can be asked, after the reset, to recreate the State +// in the new cache. For example, in a DFA method ("this" is a DFA): +// +// StateSaver saver(this, s); +// ResetCache(cache_lock); +// s = saver.Restore(); +// +// The saver should always have room in the cache to re-create the state, +// because resetting the cache locks out all other threads, and the cache +// is known to have room for at least a couple states (otherwise the DFA +// constructor fails). + +class DFA::StateSaver { + public: + explicit StateSaver(DFA* dfa, State* state); + ~StateSaver(); + + // Recreates and returns a state equivalent to the + // original state passed to the constructor. + // Returns NULL if the cache has filled, but + // since the DFA guarantees to have room in the cache + // for a couple states, should never return NULL + // if used right after ResetCache. + State* Restore(); + + private: + DFA* dfa_; // the DFA to use + int* inst_; // saved info from State + int ninst_; + uint32_t flag_; + bool is_special_; // whether original state was special + State* special_; // if is_special_, the original state + + StateSaver(const StateSaver&) = delete; + StateSaver& operator=(const StateSaver&) = delete; +}; + +DFA::StateSaver::StateSaver(DFA* dfa, State* state) { + dfa_ = dfa; + if (state <= SpecialStateMax) { + inst_ = NULL; + ninst_ = 0; + flag_ = 0; + is_special_ = true; + special_ = state; + return; + } + is_special_ = false; + special_ = NULL; + flag_ = state->flag_; + ninst_ = state->ninst_; + inst_ = new int[ninst_]; + memmove(inst_, state->inst_, ninst_*sizeof inst_[0]); +} + +DFA::StateSaver::~StateSaver() { + if (!is_special_) + delete[] inst_; +} + +DFA::State* DFA::StateSaver::Restore() { + if (is_special_) + return special_; + MutexLock l(&dfa_->mutex_); + State* s = dfa_->CachedState(inst_, ninst_, flag_); + if (s == NULL) + LOG(DFATAL) << "StateSaver failed to restore state."; + return s; +} + + +////////////////////////////////////////////////////////////////////// +// +// DFA execution. +// +// The basic search loop is easy: start in a state s and then for each +// byte c in the input, s = s->next[c]. +// +// This simple description omits a few efficiency-driven complications. +// +// First, the State graph is constructed incrementally: it is possible +// that s->next[c] is null, indicating that that state has not been +// fully explored. In this case, RunStateOnByte must be invoked to +// determine the next state, which is cached in s->next[c] to save +// future effort. An alternative reason for s->next[c] to be null is +// that the DFA has reached a so-called "dead state", in which any match +// is no longer possible. In this case RunStateOnByte will return NULL +// and the processing of the string can stop early. +// +// Second, a 256-element pointer array for s->next_ makes each State +// quite large (2kB on 64-bit machines). Instead, dfa->bytemap_[] +// maps from bytes to "byte classes" and then next_ only needs to have +// as many pointers as there are byte classes. A byte class is simply a +// range of bytes that the regexp never distinguishes between. +// A regexp looking for a[abc] would have four byte ranges -- 0 to 'a'-1, +// 'a', 'b' to 'c', and 'c' to 0xFF. The bytemap slows us a little bit +// but in exchange we typically cut the size of a State (and thus our +// memory footprint) by about 5-10x. The comments still refer to +// s->next[c] for simplicity, but code should refer to s->next_[bytemap_[c]]. +// +// Third, it is common for a DFA for an unanchored match to begin in a +// state in which only one particular byte value can take the DFA to a +// different state. That is, s->next[c] != s for only one c. In this +// situation, the DFA can do better than executing the simple loop. +// Instead, it can call memchr to search very quickly for the byte c. +// Whether the start state has this property is determined during a +// pre-compilation pass, and if so, the byte b is passed to the search +// loop as the "first_byte" argument, along with a boolean "have_first_byte". +// +// Fourth, the desired behavior is to search for the leftmost-best match +// (approximately, the same one that Perl would find), which is not +// necessarily the match ending earliest in the string. Each time a +// match is found, it must be noted, but the DFA must continue on in +// hope of finding a higher-priority match. In some cases, the caller only +// cares whether there is any match at all, not which one is found. +// The "want_earliest_match" flag causes the search to stop at the first +// match found. +// +// Fifth, one algorithm that uses the DFA needs it to run over the +// input string backward, beginning at the end and ending at the beginning. +// Passing false for the "run_forward" flag causes the DFA to run backward. +// +// The checks for these last three cases, which in a naive implementation +// would be performed once per input byte, slow the general loop enough +// to merit specialized versions of the search loop for each of the +// eight possible settings of the three booleans. Rather than write +// eight different functions, we write one general implementation and then +// inline it to create the specialized ones. +// +// Note that matches are delayed by one byte, to make it easier to +// accomodate match conditions depending on the next input byte (like $ and \b). +// When s->next[c]->IsMatch(), it means that there is a match ending just +// *before* byte c. + +// The generic search loop. Searches text for a match, returning +// the pointer to the end of the chosen match, or NULL if no match. +// The bools are equal to the same-named variables in params, but +// making them function arguments lets the inliner specialize +// this function to each combination (see two paragraphs above). +inline bool DFA::InlinedSearchLoop(SearchParams* params, + bool have_first_byte, + bool want_earliest_match, + bool run_forward) { + State* start = params->start; + const uint8_t* bp = BytePtr(params->text.begin()); // start of text + const uint8_t* p = bp; // text scanning point + const uint8_t* ep = BytePtr(params->text.end()); // end of text + const uint8_t* resetp = NULL; // p at last cache reset + if (!run_forward) { + using std::swap; + swap(p, ep); + } + + const uint8_t* bytemap = prog_->bytemap(); + const uint8_t* lastmatch = NULL; // most recent matching position in text + bool matched = false; + + State* s = start; + if (ExtraDebug) + fprintf(stderr, "@stx: %s\n", DumpState(s).c_str()); + + if (s->IsMatch()) { + matched = true; + lastmatch = p; + if (ExtraDebug) + fprintf(stderr, "match @stx! [%s]\n", DumpState(s).c_str()); + if (params->matches != NULL && kind_ == Prog::kManyMatch) { + for (int i = s->ninst_ - 1; i >= 0; i--) { + int id = s->inst_[i]; + if (id == MatchSep) + break; + params->matches->insert(id); + } + } + if (want_earliest_match) { + params->ep = reinterpret_cast(lastmatch); + return true; + } + } + + while (p != ep) { + if (ExtraDebug) + fprintf(stderr, "@%td: %s\n", + p - bp, DumpState(s).c_str()); + + if (have_first_byte && s == start) { + // In start state, only way out is to find first_byte, + // so use optimized assembly in memchr to skip ahead. + // If first_byte isn't found, we can skip to the end + // of the string. + if (run_forward) { + if ((p = BytePtr(memchr(p, params->first_byte, ep - p))) == NULL) { + p = ep; + break; + } + } else { + if ((p = BytePtr(memrchr(ep, params->first_byte, p - ep))) == NULL) { + p = ep; + break; + } + p++; + } + } + + int c; + if (run_forward) + c = *p++; + else + c = *--p; + + // Note that multiple threads might be consulting + // s->next_[bytemap[c]] simultaneously. + // RunStateOnByte takes care of the appropriate locking, + // including a memory barrier so that the unlocked access + // (sometimes known as "double-checked locking") is safe. + // The alternative would be either one DFA per thread + // or one mutex operation per input byte. + // + // ns == DeadState means the state is known to be dead + // (no more matches are possible). + // ns == NULL means the state has not yet been computed + // (need to call RunStateOnByteUnlocked). + // RunStateOnByte returns ns == NULL if it is out of memory. + // ns == FullMatchState means the rest of the string matches. + // + // Okay to use bytemap[] not ByteMap() here, because + // c is known to be an actual byte and not kByteEndText. + + State* ns = s->next_[bytemap[c]].load(std::memory_order_acquire); + if (ns == NULL) { + ns = RunStateOnByteUnlocked(s, c); + if (ns == NULL) { + // After we reset the cache, we hold cache_mutex exclusively, + // so if resetp != NULL, it means we filled the DFA state + // cache with this search alone (without any other threads). + // Benchmarks show that doing a state computation on every + // byte runs at about 0.2 MB/s, while the NFA (nfa.cc) can do the + // same at about 2 MB/s. Unless we're processing an average + // of 10 bytes per state computation, fail so that RE2 can + // fall back to the NFA. However, RE2::Set cannot fall back, + // so we just have to keep on keeping on in that case. + if (dfa_should_bail_when_slow && resetp != NULL && + static_cast(p - resetp) < 10*state_cache_.size() && + kind_ != Prog::kManyMatch) { + params->failed = true; + return false; + } + resetp = p; + + // Prepare to save start and s across the reset. + StateSaver save_start(this, start); + StateSaver save_s(this, s); + + // Discard all the States in the cache. + ResetCache(params->cache_lock); + + // Restore start and s so we can continue. + if ((start = save_start.Restore()) == NULL || + (s = save_s.Restore()) == NULL) { + // Restore already did LOG(DFATAL). + params->failed = true; + return false; + } + ns = RunStateOnByteUnlocked(s, c); + if (ns == NULL) { + LOG(DFATAL) << "RunStateOnByteUnlocked failed after ResetCache"; + params->failed = true; + return false; + } + } + } + if (ns <= SpecialStateMax) { + if (ns == DeadState) { + params->ep = reinterpret_cast(lastmatch); + return matched; + } + // FullMatchState + params->ep = reinterpret_cast(ep); + return true; + } + + s = ns; + if (s->IsMatch()) { + matched = true; + // The DFA notices the match one byte late, + // so adjust p before using it in the match. + if (run_forward) + lastmatch = p - 1; + else + lastmatch = p + 1; + if (ExtraDebug) + fprintf(stderr, "match @%td! [%s]\n", + lastmatch - bp, DumpState(s).c_str()); + if (params->matches != NULL && kind_ == Prog::kManyMatch) { + for (int i = s->ninst_ - 1; i >= 0; i--) { + int id = s->inst_[i]; + if (id == MatchSep) + break; + params->matches->insert(id); + } + } + if (want_earliest_match) { + params->ep = reinterpret_cast(lastmatch); + return true; + } + } + } + + // Process one more byte to see if it triggers a match. + // (Remember, matches are delayed one byte.) + if (ExtraDebug) + fprintf(stderr, "@etx: %s\n", DumpState(s).c_str()); + + int lastbyte; + if (run_forward) { + if (params->text.end() == params->context.end()) + lastbyte = kByteEndText; + else + lastbyte = params->text.end()[0] & 0xFF; + } else { + if (params->text.begin() == params->context.begin()) + lastbyte = kByteEndText; + else + lastbyte = params->text.begin()[-1] & 0xFF; + } + + State* ns = s->next_[ByteMap(lastbyte)].load(std::memory_order_acquire); + if (ns == NULL) { + ns = RunStateOnByteUnlocked(s, lastbyte); + if (ns == NULL) { + StateSaver save_s(this, s); + ResetCache(params->cache_lock); + if ((s = save_s.Restore()) == NULL) { + params->failed = true; + return false; + } + ns = RunStateOnByteUnlocked(s, lastbyte); + if (ns == NULL) { + LOG(DFATAL) << "RunStateOnByteUnlocked failed after Reset"; + params->failed = true; + return false; + } + } + } + if (ns <= SpecialStateMax) { + if (ns == DeadState) { + params->ep = reinterpret_cast(lastmatch); + return matched; + } + // FullMatchState + params->ep = reinterpret_cast(ep); + return true; + } + + s = ns; + if (s->IsMatch()) { + matched = true; + lastmatch = p; + if (ExtraDebug) + fprintf(stderr, "match @etx! [%s]\n", DumpState(s).c_str()); + if (params->matches != NULL && kind_ == Prog::kManyMatch) { + for (int i = s->ninst_ - 1; i >= 0; i--) { + int id = s->inst_[i]; + if (id == MatchSep) + break; + params->matches->insert(id); + } + } + } + + params->ep = reinterpret_cast(lastmatch); + return matched; +} + +// Inline specializations of the general loop. +bool DFA::SearchFFF(SearchParams* params) { + return InlinedSearchLoop(params, 0, 0, 0); +} +bool DFA::SearchFFT(SearchParams* params) { + return InlinedSearchLoop(params, 0, 0, 1); +} +bool DFA::SearchFTF(SearchParams* params) { + return InlinedSearchLoop(params, 0, 1, 0); +} +bool DFA::SearchFTT(SearchParams* params) { + return InlinedSearchLoop(params, 0, 1, 1); +} +bool DFA::SearchTFF(SearchParams* params) { + return InlinedSearchLoop(params, 1, 0, 0); +} +bool DFA::SearchTFT(SearchParams* params) { + return InlinedSearchLoop(params, 1, 0, 1); +} +bool DFA::SearchTTF(SearchParams* params) { + return InlinedSearchLoop(params, 1, 1, 0); +} +bool DFA::SearchTTT(SearchParams* params) { + return InlinedSearchLoop(params, 1, 1, 1); +} + +// For debugging, calls the general code directly. +bool DFA::SlowSearchLoop(SearchParams* params) { + return InlinedSearchLoop(params, + params->first_byte >= 0, + params->want_earliest_match, + params->run_forward); +} + +// For performance, calls the appropriate specialized version +// of InlinedSearchLoop. +bool DFA::FastSearchLoop(SearchParams* params) { + // Because the methods are private, the Searches array + // cannot be declared at top level. + static bool (DFA::*Searches[])(SearchParams*) = { + &DFA::SearchFFF, + &DFA::SearchFFT, + &DFA::SearchFTF, + &DFA::SearchFTT, + &DFA::SearchTFF, + &DFA::SearchTFT, + &DFA::SearchTTF, + &DFA::SearchTTT, + }; + + bool have_first_byte = params->first_byte >= 0; + int index = 4 * have_first_byte + + 2 * params->want_earliest_match + + 1 * params->run_forward; + return (this->*Searches[index])(params); +} + + +// The discussion of DFA execution above ignored the question of how +// to determine the initial state for the search loop. There are two +// factors that influence the choice of start state. +// +// The first factor is whether the search is anchored or not. +// The regexp program (Prog*) itself has +// two different entry points: one for anchored searches and one for +// unanchored searches. (The unanchored version starts with a leading ".*?" +// and then jumps to the anchored one.) +// +// The second factor is where text appears in the larger context, which +// determines which empty-string operators can be matched at the beginning +// of execution. If text is at the very beginning of context, \A and ^ match. +// Otherwise if text is at the beginning of a line, then ^ matches. +// Otherwise it matters whether the character before text is a word character +// or a non-word character. +// +// The two cases (unanchored vs not) and four cases (empty-string flags) +// combine to make the eight cases recorded in the DFA's begin_text_[2], +// begin_line_[2], after_wordchar_[2], and after_nonwordchar_[2] cached +// StartInfos. The start state for each is filled in the first time it +// is used for an actual search. + +// Examines text, context, and anchored to determine the right start +// state for the DFA search loop. Fills in params and returns true on success. +// Returns false on failure. +bool DFA::AnalyzeSearch(SearchParams* params) { + const StringPiece& text = params->text; + const StringPiece& context = params->context; + + // Sanity check: make sure that text lies within context. + if (text.begin() < context.begin() || text.end() > context.end()) { + LOG(DFATAL) << "context does not contain text"; + params->start = DeadState; + return true; + } + + // Determine correct search type. + int start; + uint32_t flags; + if (params->run_forward) { + if (text.begin() == context.begin()) { + start = kStartBeginText; + flags = kEmptyBeginText|kEmptyBeginLine; + } else if (text.begin()[-1] == '\n') { + start = kStartBeginLine; + flags = kEmptyBeginLine; + } else if (Prog::IsWordChar(text.begin()[-1] & 0xFF)) { + start = kStartAfterWordChar; + flags = kFlagLastWord; + } else { + start = kStartAfterNonWordChar; + flags = 0; + } + } else { + if (text.end() == context.end()) { + start = kStartBeginText; + flags = kEmptyBeginText|kEmptyBeginLine; + } else if (text.end()[0] == '\n') { + start = kStartBeginLine; + flags = kEmptyBeginLine; + } else if (Prog::IsWordChar(text.end()[0] & 0xFF)) { + start = kStartAfterWordChar; + flags = kFlagLastWord; + } else { + start = kStartAfterNonWordChar; + flags = 0; + } + } + if (params->anchored) + start |= kStartAnchored; + StartInfo* info = &start_[start]; + + // Try once without cache_lock for writing. + // Try again after resetting the cache + // (ResetCache will relock cache_lock for writing). + if (!AnalyzeSearchHelper(params, info, flags)) { + ResetCache(params->cache_lock); + if (!AnalyzeSearchHelper(params, info, flags)) { + LOG(DFATAL) << "Failed to analyze start state."; + params->failed = true; + return false; + } + } + + if (ExtraDebug) + fprintf(stderr, "anchored=%d fwd=%d flags=%#x state=%s first_byte=%d\n", + params->anchored, params->run_forward, flags, + DumpState(info->start).c_str(), info->first_byte.load()); + + params->start = info->start; + params->first_byte = info->first_byte.load(std::memory_order_acquire); + + return true; +} + +// Fills in info if needed. Returns true on success, false on failure. +bool DFA::AnalyzeSearchHelper(SearchParams* params, StartInfo* info, + uint32_t flags) { + // Quick check. + int fb = info->first_byte.load(std::memory_order_acquire); + if (fb != kFbUnknown) + return true; + + MutexLock l(&mutex_); + fb = info->first_byte.load(std::memory_order_relaxed); + if (fb != kFbUnknown) + return true; + + q0_->clear(); + AddToQueue(q0_, + params->anchored ? prog_->start() : prog_->start_unanchored(), + flags); + info->start = WorkqToCachedState(q0_, NULL, flags); + if (info->start == NULL) + return false; + + if (info->start == DeadState) { + // Synchronize with "quick check" above. + info->first_byte.store(kFbNone, std::memory_order_release); + return true; + } + + if (info->start == FullMatchState) { + // Synchronize with "quick check" above. + info->first_byte.store(kFbNone, std::memory_order_release); // will be ignored + return true; + } + + // Even if we have a first_byte, we cannot use it when anchored and, + // less obviously, we cannot use it when we are going to need flags. + // This trick works only when there is a single byte that leads to a + // different state! + int first_byte = prog_->first_byte(); + if (first_byte == -1 || + params->anchored || + info->start->flag_ >> kFlagNeedShift != 0) + first_byte = kFbNone; + + // Synchronize with "quick check" above. + info->first_byte.store(first_byte, std::memory_order_release); + return true; +} + +// The actual DFA search: calls AnalyzeSearch and then FastSearchLoop. +bool DFA::Search(const StringPiece& text, + const StringPiece& context, + bool anchored, + bool want_earliest_match, + bool run_forward, + bool* failed, + const char** epp, + SparseSet* matches) { + *epp = NULL; + if (!ok()) { + *failed = true; + return false; + } + *failed = false; + + if (ExtraDebug) { + fprintf(stderr, "\nprogram:\n%s\n", prog_->DumpUnanchored().c_str()); + fprintf(stderr, "text %s anchored=%d earliest=%d fwd=%d kind %d\n", + std::string(text).c_str(), anchored, want_earliest_match, + run_forward, kind_); + } + + RWLocker l(&cache_mutex_); + SearchParams params(text, context, &l); + params.anchored = anchored; + params.want_earliest_match = want_earliest_match; + params.run_forward = run_forward; + params.matches = matches; + + if (!AnalyzeSearch(¶ms)) { + *failed = true; + return false; + } + if (params.start == DeadState) + return false; + if (params.start == FullMatchState) { + if (run_forward == want_earliest_match) + *epp = text.begin(); + else + *epp = text.end(); + return true; + } + if (ExtraDebug) + fprintf(stderr, "start %s\n", DumpState(params.start).c_str()); + bool ret = FastSearchLoop(¶ms); + if (params.failed) { + *failed = true; + return false; + } + *epp = params.ep; + return ret; +} + +DFA* Prog::GetDFA(MatchKind kind) { + // For a forward DFA, half the memory goes to each DFA. + // However, if it is a "many match" DFA, then there is + // no counterpart with which the memory must be shared. + // + // For a reverse DFA, all the memory goes to the + // "longest match" DFA, because RE2 never does reverse + // "first match" searches. + if (kind == kFirstMatch) { + std::call_once(dfa_first_once_, [](Prog* prog) { + prog->dfa_first_ = new DFA(prog, kFirstMatch, prog->dfa_mem_ / 2); + }, this); + return dfa_first_; + } else if (kind == kManyMatch) { + std::call_once(dfa_first_once_, [](Prog* prog) { + prog->dfa_first_ = new DFA(prog, kManyMatch, prog->dfa_mem_); + }, this); + return dfa_first_; + } else { + std::call_once(dfa_longest_once_, [](Prog* prog) { + if (!prog->reversed_) + prog->dfa_longest_ = new DFA(prog, kLongestMatch, prog->dfa_mem_ / 2); + else + prog->dfa_longest_ = new DFA(prog, kLongestMatch, prog->dfa_mem_); + }, this); + return dfa_longest_; + } +} + +void Prog::DeleteDFA(DFA* dfa) { + delete dfa; +} + +// Executes the regexp program to search in text, +// which itself is inside the larger context. (As a convenience, +// passing a NULL context is equivalent to passing text.) +// Returns true if a match is found, false if not. +// If a match is found, fills in match0->end() to point at the end of the match +// and sets match0->begin() to text.begin(), since the DFA can't track +// where the match actually began. +// +// This is the only external interface (class DFA only exists in this file). +// +bool Prog::SearchDFA(const StringPiece& text, const StringPiece& const_context, + Anchor anchor, MatchKind kind, StringPiece* match0, + bool* failed, SparseSet* matches) { + *failed = false; + + StringPiece context = const_context; + if (context.begin() == NULL) + context = text; + bool carat = anchor_start(); + bool dollar = anchor_end(); + if (reversed_) { + using std::swap; + swap(carat, dollar); + } + if (carat && context.begin() != text.begin()) + return false; + if (dollar && context.end() != text.end()) + return false; + + // Handle full match by running an anchored longest match + // and then checking if it covers all of text. + bool anchored = anchor == kAnchored || anchor_start() || kind == kFullMatch; + bool endmatch = false; + if (kind == kManyMatch) { + // This is split out in order to avoid clobbering kind. + } else if (kind == kFullMatch || anchor_end()) { + endmatch = true; + kind = kLongestMatch; + } + + // If the caller doesn't care where the match is (just whether one exists), + // then we can stop at the very first match we find, the so-called + // "earliest match". + bool want_earliest_match = false; + if (kind == kManyMatch) { + // This is split out in order to avoid clobbering kind. + if (matches == NULL) { + want_earliest_match = true; + } + } else if (match0 == NULL && !endmatch) { + want_earliest_match = true; + kind = kLongestMatch; + } + + DFA* dfa = GetDFA(kind); + const char* ep; + bool matched = dfa->Search(text, context, anchored, + want_earliest_match, !reversed_, + failed, &ep, matches); + if (*failed) + return false; + if (!matched) + return false; + if (endmatch && ep != (reversed_ ? text.begin() : text.end())) + return false; + + // If caller cares, record the boundary of the match. + // We only know where it ends, so use the boundary of text + // as the beginning. + if (match0) { + if (reversed_) + *match0 = StringPiece(ep, static_cast(text.end() - ep)); + else + *match0 = + StringPiece(text.begin(), static_cast(ep - text.begin())); + } + return true; +} + +// Build out all states in DFA. Returns number of states. +int DFA::BuildAllStates(const Prog::DFAStateCallback& cb) { + if (!ok()) + return 0; + + // Pick out start state for unanchored search + // at beginning of text. + RWLocker l(&cache_mutex_); + SearchParams params(StringPiece(), StringPiece(), &l); + params.anchored = false; + if (!AnalyzeSearch(¶ms) || + params.start == NULL || + params.start == DeadState) + return 0; + + // Add start state to work queue. + // Note that any State* that we handle here must point into the cache, + // so we can simply depend on pointer-as-a-number hashing and equality. + std::unordered_map m; + std::deque q; + m.emplace(params.start, static_cast(m.size())); + q.push_back(params.start); + + // Compute the input bytes needed to cover all of the next pointers. + int nnext = prog_->bytemap_range() + 1; // + 1 for kByteEndText slot + std::vector input(nnext); + for (int c = 0; c < 256; c++) { + int b = prog_->bytemap()[c]; + while (c < 256-1 && prog_->bytemap()[c+1] == b) + c++; + input[b] = c; + } + input[prog_->bytemap_range()] = kByteEndText; + + // Scratch space for the output. + std::vector output(nnext); + + // Flood to expand every state. + bool oom = false; + while (!q.empty()) { + State* s = q.front(); + q.pop_front(); + for (int c : input) { + State* ns = RunStateOnByteUnlocked(s, c); + if (ns == NULL) { + oom = true; + break; + } + if (ns == DeadState) { + output[ByteMap(c)] = -1; + continue; + } + if (m.find(ns) == m.end()) { + m.emplace(ns, static_cast(m.size())); + q.push_back(ns); + } + output[ByteMap(c)] = m[ns]; + } + if (cb) + cb(oom ? NULL : output.data(), + s == FullMatchState || s->IsMatch()); + if (oom) + break; + } + + return static_cast(m.size()); +} + +// Build out all states in DFA for kind. Returns number of states. +int Prog::BuildEntireDFA(MatchKind kind, const DFAStateCallback& cb) { + return GetDFA(kind)->BuildAllStates(cb); +} + +void Prog::TEST_dfa_should_bail_when_slow(bool b) { + dfa_should_bail_when_slow = b; +} + +// Computes min and max for matching string. +// Won't return strings bigger than maxlen. +bool DFA::PossibleMatchRange(std::string* min, std::string* max, int maxlen) { + if (!ok()) + return false; + + // NOTE: if future users of PossibleMatchRange want more precision when + // presented with infinitely repeated elements, consider making this a + // parameter to PossibleMatchRange. + static int kMaxEltRepetitions = 0; + + // Keep track of the number of times we've visited states previously. We only + // revisit a given state if it's part of a repeated group, so if the value + // portion of the map tuple exceeds kMaxEltRepetitions we bail out and set + // |*max| to |PrefixSuccessor(*max)|. + // + // Also note that previously_visited_states[UnseenStatePtr] will, in the STL + // tradition, implicitly insert a '0' value at first use. We take advantage + // of that property below. + std::unordered_map previously_visited_states; + + // Pick out start state for anchored search at beginning of text. + RWLocker l(&cache_mutex_); + SearchParams params(StringPiece(), StringPiece(), &l); + params.anchored = true; + if (!AnalyzeSearch(¶ms)) + return false; + if (params.start == DeadState) { // No matching strings + *min = ""; + *max = ""; + return true; + } + if (params.start == FullMatchState) // Every string matches: no max + return false; + + // The DFA is essentially a big graph rooted at params.start, + // and paths in the graph correspond to accepted strings. + // Each node in the graph has potentially 256+1 arrows + // coming out, one for each byte plus the magic end of + // text character kByteEndText. + + // To find the smallest possible prefix of an accepted + // string, we just walk the graph preferring to follow + // arrows with the lowest bytes possible. To find the + // largest possible prefix, we follow the largest bytes + // possible. + + // The test for whether there is an arrow from s on byte j is + // ns = RunStateOnByteUnlocked(s, j); + // if (ns == NULL) + // return false; + // if (ns != DeadState && ns->ninst > 0) + // The RunStateOnByteUnlocked call asks the DFA to build out the graph. + // It returns NULL only if the DFA has run out of memory, + // in which case we can't be sure of anything. + // The second check sees whether there was graph built + // and whether it is interesting graph. Nodes might have + // ns->ninst == 0 if they exist only to represent the fact + // that a match was found on the previous byte. + + // Build minimum prefix. + State* s = params.start; + min->clear(); + MutexLock lock(&mutex_); + for (int i = 0; i < maxlen; i++) { + if (previously_visited_states[s] > kMaxEltRepetitions) + break; + previously_visited_states[s]++; + + // Stop if min is a match. + State* ns = RunStateOnByte(s, kByteEndText); + if (ns == NULL) // DFA out of memory + return false; + if (ns != DeadState && (ns == FullMatchState || ns->IsMatch())) + break; + + // Try to extend the string with low bytes. + bool extended = false; + for (int j = 0; j < 256; j++) { + ns = RunStateOnByte(s, j); + if (ns == NULL) // DFA out of memory + return false; + if (ns == FullMatchState || + (ns > SpecialStateMax && ns->ninst_ > 0)) { + extended = true; + min->append(1, static_cast(j)); + s = ns; + break; + } + } + if (!extended) + break; + } + + // Build maximum prefix. + previously_visited_states.clear(); + s = params.start; + max->clear(); + for (int i = 0; i < maxlen; i++) { + if (previously_visited_states[s] > kMaxEltRepetitions) + break; + previously_visited_states[s] += 1; + + // Try to extend the string with high bytes. + bool extended = false; + for (int j = 255; j >= 0; j--) { + State* ns = RunStateOnByte(s, j); + if (ns == NULL) + return false; + if (ns == FullMatchState || + (ns > SpecialStateMax && ns->ninst_ > 0)) { + extended = true; + max->append(1, static_cast(j)); + s = ns; + break; + } + } + if (!extended) { + // Done, no need for PrefixSuccessor. + return true; + } + } + + // Stopped while still adding to *max - round aaaaaaaaaa... to aaaa...b + PrefixSuccessor(max); + + // If there are no bytes left, we have no way to say "there is no maximum + // string". We could make the interface more complicated and be able to + // return "there is no maximum but here is a minimum", but that seems like + // overkill -- the most common no-max case is all possible strings, so not + // telling the caller that the empty string is the minimum match isn't a + // great loss. + if (max->empty()) + return false; + + return true; +} + +// PossibleMatchRange for a Prog. +bool Prog::PossibleMatchRange(std::string* min, std::string* max, int maxlen) { + // Have to use dfa_longest_ to get all strings for full matches. + // For example, (a|aa) never matches aa in first-match mode. + return GetDFA(kLongestMatch)->PossibleMatchRange(min, max, maxlen); +} + +} // namespace re2 diff --git a/extern/re2/re2/filtered_re2.cc b/extern/re2/re2/filtered_re2.cc new file mode 100644 index 0000000000..e5d8de5ce6 --- /dev/null +++ b/extern/re2/re2/filtered_re2.cc @@ -0,0 +1,121 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "re2/filtered_re2.h" + +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "re2/prefilter.h" +#include "re2/prefilter_tree.h" + +namespace re2 { + +FilteredRE2::FilteredRE2() + : compiled_(false), + prefilter_tree_(new PrefilterTree()) { +} + +FilteredRE2::FilteredRE2(int min_atom_len) + : compiled_(false), + prefilter_tree_(new PrefilterTree(min_atom_len)) { +} + +FilteredRE2::~FilteredRE2() { + for (size_t i = 0; i < re2_vec_.size(); i++) + delete re2_vec_[i]; + delete prefilter_tree_; +} + +RE2::ErrorCode FilteredRE2::Add(const StringPiece& pattern, + const RE2::Options& options, int* id) { + RE2* re = new RE2(pattern, options); + RE2::ErrorCode code = re->error_code(); + + if (!re->ok()) { + if (options.log_errors()) { + LOG(ERROR) << "Couldn't compile regular expression, skipping: " + << re << " due to error " << re->error(); + } + delete re; + } else { + *id = static_cast(re2_vec_.size()); + re2_vec_.push_back(re); + } + + return code; +} + +void FilteredRE2::Compile(std::vector* atoms) { + if (compiled_) { + LOG(ERROR) << "Compile called already."; + return; + } + + if (re2_vec_.empty()) { + LOG(ERROR) << "Compile called before Add."; + return; + } + + for (size_t i = 0; i < re2_vec_.size(); i++) { + Prefilter* prefilter = Prefilter::FromRE2(re2_vec_[i]); + prefilter_tree_->Add(prefilter); + } + atoms->clear(); + prefilter_tree_->Compile(atoms); + compiled_ = true; +} + +int FilteredRE2::SlowFirstMatch(const StringPiece& text) const { + for (size_t i = 0; i < re2_vec_.size(); i++) + if (RE2::PartialMatch(text, *re2_vec_[i])) + return static_cast(i); + return -1; +} + +int FilteredRE2::FirstMatch(const StringPiece& text, + const std::vector& atoms) const { + if (!compiled_) { + LOG(DFATAL) << "FirstMatch called before Compile."; + return -1; + } + std::vector regexps; + prefilter_tree_->RegexpsGivenStrings(atoms, ®exps); + for (size_t i = 0; i < regexps.size(); i++) + if (RE2::PartialMatch(text, *re2_vec_[regexps[i]])) + return regexps[i]; + return -1; +} + +bool FilteredRE2::AllMatches( + const StringPiece& text, + const std::vector& atoms, + std::vector* matching_regexps) const { + matching_regexps->clear(); + std::vector regexps; + prefilter_tree_->RegexpsGivenStrings(atoms, ®exps); + for (size_t i = 0; i < regexps.size(); i++) + if (RE2::PartialMatch(text, *re2_vec_[regexps[i]])) + matching_regexps->push_back(regexps[i]); + return !matching_regexps->empty(); +} + +void FilteredRE2::AllPotentials( + const std::vector& atoms, + std::vector* potential_regexps) const { + prefilter_tree_->RegexpsGivenStrings(atoms, potential_regexps); +} + +void FilteredRE2::RegexpsGivenStrings(const std::vector& matched_atoms, + std::vector* passed_regexps) { + prefilter_tree_->RegexpsGivenStrings(matched_atoms, passed_regexps); +} + +void FilteredRE2::PrintPrefilter(int regexpid) { + prefilter_tree_->PrintPrefilter(regexpid); +} + +} // namespace re2 diff --git a/extern/re2/re2/filtered_re2.h b/extern/re2/re2/filtered_re2.h new file mode 100644 index 0000000000..4118accc87 --- /dev/null +++ b/extern/re2/re2/filtered_re2.h @@ -0,0 +1,109 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_FILTERED_RE2_H_ +#define RE2_FILTERED_RE2_H_ + +// The class FilteredRE2 is used as a wrapper to multiple RE2 regexps. +// It provides a prefilter mechanism that helps in cutting down the +// number of regexps that need to be actually searched. +// +// By design, it does not include a string matching engine. This is to +// allow the user of the class to use their favorite string match +// engine. The overall flow is: Add all the regexps using Add, then +// Compile the FilteredRE2. The compile returns strings that need to +// be matched. Note that all returned strings are lowercase. For +// applying regexps to a search text, the caller does the string +// matching using the strings returned. When doing the string match, +// note that the caller has to do that on lower cased version of the +// search text. Then call FirstMatch or AllMatches with a vector of +// indices of strings that were found in the text to get the actual +// regexp matches. + +#include +#include + +#include "re2/re2.h" + +namespace re2 { + +class PrefilterTree; + +class FilteredRE2 { + public: + FilteredRE2(); + explicit FilteredRE2(int min_atom_len); + ~FilteredRE2(); + + // Uses RE2 constructor to create a RE2 object (re). Returns + // re->error_code(). If error_code is other than NoError, then re is + // deleted and not added to re2_vec_. + RE2::ErrorCode Add(const StringPiece& pattern, + const RE2::Options& options, + int *id); + + // Prepares the regexps added by Add for filtering. Returns a set + // of strings that the caller should check for in candidate texts. + // The returned strings are lowercased. When doing string matching, + // the search text should be lowercased first to find matching + // strings from the set of strings returned by Compile. Call after + // all Add calls are done. + void Compile(std::vector* strings_to_match); + + // Returns the index of the first matching regexp. + // Returns -1 on no match. Can be called prior to Compile. + // Does not do any filtering: simply tries to Match the + // regexps in a loop. + int SlowFirstMatch(const StringPiece& text) const; + + // Returns the index of the first matching regexp. + // Returns -1 on no match. Compile has to be called before + // calling this. + int FirstMatch(const StringPiece& text, + const std::vector& atoms) const; + + // Returns the indices of all matching regexps, after first clearing + // matched_regexps. + bool AllMatches(const StringPiece& text, + const std::vector& atoms, + std::vector* matching_regexps) const; + + // Returns the indices of all potentially matching regexps after first + // clearing potential_regexps. + // A regexp is potentially matching if it passes the filter. + // If a regexp passes the filter it may still not match. + // A regexp that does not pass the filter is guaranteed to not match. + void AllPotentials(const std::vector& atoms, + std::vector* potential_regexps) const; + + // The number of regexps added. + int NumRegexps() const { return static_cast(re2_vec_.size()); } + + // Get the individual RE2 objects. + const RE2& GetRE2(int regexpid) const { return *re2_vec_[regexpid]; } + + private: + // Print prefilter. + void PrintPrefilter(int regexpid); + + // Useful for testing and debugging. + void RegexpsGivenStrings(const std::vector& matched_atoms, + std::vector* passed_regexps); + + // All the regexps in the FilteredRE2. + std::vector re2_vec_; + + // Has the FilteredRE2 been compiled using Compile() + bool compiled_; + + // An AND-OR tree of string atoms used for filtering regexps. + PrefilterTree* prefilter_tree_; + + FilteredRE2(const FilteredRE2&) = delete; + FilteredRE2& operator=(const FilteredRE2&) = delete; +}; + +} // namespace re2 + +#endif // RE2_FILTERED_RE2_H_ diff --git a/extern/re2/re2/fuzzing/re2_fuzzer.cc b/extern/re2/re2/fuzzing/re2_fuzzer.cc new file mode 100644 index 0000000000..061c418e18 --- /dev/null +++ b/extern/re2/re2/fuzzing/re2_fuzzer.cc @@ -0,0 +1,173 @@ +// Copyright 2016 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include +#include +#include +#include +#include + +#include "re2/prefilter.h" +#include "re2/re2.h" + +using re2::StringPiece; + +// NOT static, NOT signed. +uint8_t dummy = 0; + +void Test(StringPiece pattern, const RE2::Options& options, StringPiece text) { + RE2 re(pattern, options); + if (!re.ok()) + return; + + // Don't waste time fuzzing programs with large substrings. + // They can cause bug reports due to fuzzer timeouts when they + // are repetitions (e.g. hundreds of NUL bytes) and matching is + // unanchored. And they aren't interesting for fuzzing purposes. + std::unique_ptr prefilter(re2::Prefilter::FromRE2(&re)); + if (prefilter == nullptr) + return; + std::queue nodes; + nodes.push(prefilter.get()); + while (!nodes.empty()) { + re2::Prefilter* node = nodes.front(); + nodes.pop(); + if (node->op() == re2::Prefilter::ATOM) { + if (node->atom().size() > 9) + return; + } else if (node->op() == re2::Prefilter::AND || + node->op() == re2::Prefilter::OR) { + for (re2::Prefilter* sub : *node->subs()) + nodes.push(sub); + } + } + + // Don't waste time fuzzing high-size programs. + // They can cause bug reports due to fuzzer timeouts. + int size = re.ProgramSize(); + if (size > 9999) + return; + int rsize = re.ReverseProgramSize(); + if (rsize > 9999) + return; + + // Don't waste time fuzzing high-fanout programs. + // They can cause bug reports due to fuzzer timeouts. + std::map histogram; + int fanout = re.ProgramFanout(&histogram); + if (fanout > 9) + return; + int rfanout = re.ReverseProgramFanout(&histogram); + if (rfanout > 9) + return; + + if (re.NumberOfCapturingGroups() == 0) { + // Avoid early return due to too many arguments. + StringPiece sp = text; + RE2::FullMatch(sp, re); + RE2::PartialMatch(sp, re); + RE2::Consume(&sp, re); + sp = text; // Reset. + RE2::FindAndConsume(&sp, re); + } else { + // Okay, we have at least one capturing group... + // Try conversion for variously typed arguments. + StringPiece sp = text; + short s; + RE2::FullMatch(sp, re, &s); + long l; + RE2::PartialMatch(sp, re, &l); + float f; + RE2::Consume(&sp, re, &f); + sp = text; // Reset. + double d; + RE2::FindAndConsume(&sp, re, &d); + } + + std::string s = std::string(text); + RE2::Replace(&s, re, ""); + s = std::string(text); // Reset. + RE2::GlobalReplace(&s, re, ""); + + std::string min, max; + re.PossibleMatchRange(&min, &max, /*maxlen=*/9); + + // Exercise some other API functionality. + dummy += re.NamedCapturingGroups().size(); + dummy += re.CapturingGroupNames().size(); + dummy += RE2::QuoteMeta(pattern).size(); +} + +// Entry point for libFuzzer. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + if (size == 0 || size > 999) + return 0; + + // Crudely limit the use of ., \p, \P, \d, \D, \s, \S, \w and \W. + // Otherwise, we will waste time on inputs that have long runs of various + // character classes. The fuzzer has shown itself to be easily capable of + // generating such patterns that fall within the other limits, but result + // in timeouts nonetheless. The marginal cost is high - even more so when + // counted repetition is involved - whereas the marginal benefit is zero. + // TODO(junyer): Handle [:isalnum:] et al. when they start to cause pain. + int char_class = 0; + int backslash_p = 0; // very expensive, so handle specially + for (size_t i = 0; i < size; i++) { + if (data[i] == '.') + char_class++; + if (data[i] != '\\') + continue; + i++; + if (i >= size) + break; + if (data[i] == 'p' || data[i] == 'P' || + data[i] == 'd' || data[i] == 'D' || + data[i] == 's' || data[i] == 'S' || + data[i] == 'w' || data[i] == 'W') + char_class++; + if (data[i] == 'p' || data[i] == 'P') + backslash_p++; + } + if (char_class > 9) + return 0; + if (backslash_p > 1) + return 0; + + // The one-at-a-time hash by Bob Jenkins. + uint32_t hash = 0; + for (size_t i = 0; i < size; i++) { + hash += data[i]; + hash += (hash << 10); + hash ^= (hash >> 6); + } + hash += (hash << 3); + hash ^= (hash >> 11); + hash += (hash << 15); + + RE2::Options options; + options.set_log_errors(false); + options.set_max_mem(64 << 20); + options.set_encoding(hash & 1 ? RE2::Options::EncodingLatin1 + : RE2::Options::EncodingUTF8); + options.set_posix_syntax(hash & 2); + options.set_longest_match(hash & 4); + options.set_literal(hash & 8); + options.set_never_nl(hash & 16); + options.set_dot_nl(hash & 32); + options.set_never_capture(hash & 64); + options.set_case_sensitive(hash & 128); + options.set_perl_classes(hash & 256); + options.set_word_boundary(hash & 512); + options.set_one_line(hash & 1024); + + const char* ptr = reinterpret_cast(data); + int len = static_cast(size); + + StringPiece pattern(ptr, len); + StringPiece text(ptr, len); + Test(pattern, options, text); + + return 0; +} diff --git a/extern/re2/re2/make_perl_groups.pl b/extern/re2/re2/make_perl_groups.pl new file mode 100644 index 0000000000..d9fcdafaaf --- /dev/null +++ b/extern/re2/re2/make_perl_groups.pl @@ -0,0 +1,116 @@ +#!/usr/bin/perl +# Copyright 2008 The RE2 Authors. All Rights Reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Generate table entries giving character ranges +# for POSIX/Perl character classes. Rather than +# figure out what the definition is, it is easier to ask +# Perl about each letter from 0-128 and write down +# its answer. + +@posixclasses = ( + "[:alnum:]", + "[:alpha:]", + "[:ascii:]", + "[:blank:]", + "[:cntrl:]", + "[:digit:]", + "[:graph:]", + "[:lower:]", + "[:print:]", + "[:punct:]", + "[:space:]", + "[:upper:]", + "[:word:]", + "[:xdigit:]", +); + +@perlclasses = ( + "\\d", + "\\s", + "\\w", +); + +%overrides = ( + # Prior to Perl 5.18, \s did not match vertical tab. + # RE2 preserves that original behaviour. + "\\s:11" => 0, +); + +sub ComputeClass($) { + my ($cname) = @_; + my @ranges; + my $regexp = qr/[$cname]/; + my $start = -1; + for (my $i=0; $i<=129; $i++) { + if ($i == 129) { $i = 256; } + if ($i <= 128 && ($overrides{"$cname:$i"} // chr($i) =~ $regexp)) { + if ($start < 0) { + $start = $i; + } + } else { + if ($start >= 0) { + push @ranges, [$start, $i-1]; + } + $start = -1; + } + } + return @ranges; +} + +sub PrintClass($$@) { + my ($cnum, $cname, @ranges) = @_; + print "static const URange16 code${cnum}[] = { /* $cname */\n"; + for (my $i=0; $i<@ranges; $i++) { + my @a = @{$ranges[$i]}; + printf "\t{ 0x%x, 0x%x },\n", $a[0], $a[1]; + } + print "};\n"; + my $n = @ranges; + my $escname = $cname; + $escname =~ s/\\/\\\\/g; + $negname = $escname; + if ($negname =~ /:/) { + $negname =~ s/:/:^/; + } else { + $negname =~ y/a-z/A-Z/; + } + return "{ \"$escname\", +1, code$cnum, $n }", "{ \"$negname\", -1, code$cnum, $n }"; +} + +my $cnum = 0; + +sub PrintClasses($@) { + my ($pname, @classes) = @_; + my @entries; + foreach my $cname (@classes) { + my @ranges = ComputeClass($cname); + push @entries, PrintClass(++$cnum, $cname, @ranges); + } + print "const UGroup ${pname}_groups[] = {\n"; + foreach my $e (@entries) { + print "\t$e,\n"; + } + print "};\n"; + my $count = @entries; + print "const int num_${pname}_groups = $count;\n"; +} + +print <perl_groups.cc + +#include "re2/unicode_groups.h" + +namespace re2 { + +EOF + +PrintClasses("perl", @perlclasses); +PrintClasses("posix", @posixclasses); + +print <unicode_casefold.cc + +#include "re2/unicode_casefold.h" + +namespace re2 { + +""" + +_trailer = """ + +} // namespace re2 + +""" + +def _Delta(a, b): + """Compute the delta for b - a. Even/odd and odd/even + are handled specially, as described above.""" + if a+1 == b: + if a%2 == 0: + return 'EvenOdd' + else: + return 'OddEven' + if a == b+1: + if a%2 == 0: + return 'OddEven' + else: + return 'EvenOdd' + return b - a + +def _AddDelta(a, delta): + """Return a + delta, handling EvenOdd and OddEven specially.""" + if type(delta) == int: + return a+delta + if delta == 'EvenOdd': + if a%2 == 0: + return a+1 + else: + return a-1 + if delta == 'OddEven': + if a%2 == 1: + return a+1 + else: + return a-1 + print("Bad Delta:", delta, file=sys.stderr) + raise unicode.Error("Bad Delta") + +def _MakeRanges(pairs): + """Turn a list like [(65,97), (66, 98), ..., (90,122)] + into [(65, 90, +32)].""" + ranges = [] + last = -100 + + def evenodd(last, a, b, r): + if a != last+1 or b != _AddDelta(a, r[2]): + return False + r[1] = a + return True + + def evenoddpair(last, a, b, r): + if a != last+2: + return False + delta = r[2] + d = delta + if type(delta) is not str: + return False + if delta.endswith('Skip'): + d = delta[:-4] + else: + delta = d + 'Skip' + if b != _AddDelta(a, d): + return False + r[1] = a + r[2] = delta + return True + + for a, b in pairs: + if ranges and evenodd(last, a, b, ranges[-1]): + pass + elif ranges and evenoddpair(last, a, b, ranges[-1]): + pass + else: + ranges.append([a, a, _Delta(a, b)]) + last = a + return ranges + +# The maximum size of a case-folding group. +# Case folding is implemented in parse.cc by a recursive process +# with a recursion depth equal to the size of the largest +# case-folding group, so it is important that this bound be small. +# The current tables have no group bigger than 4. +# If there are ever groups bigger than 10 or so, it will be +# time to rework the code in parse.cc. +MaxCasefoldGroup = 4 + +def main(): + lowergroups, casegroups = unicode.CaseGroups() + foldpairs = [] + seen = {} + for c in casegroups: + if len(c) > MaxCasefoldGroup: + raise unicode.Error("casefold group too long: %s" % (c,)) + for i in range(len(c)): + if c[i-1] in seen: + raise unicode.Error("bad casegroups %d -> %d" % (c[i-1], c[i])) + seen[c[i-1]] = True + foldpairs.append([c[i-1], c[i]]) + + lowerpairs = [] + for lower, group in lowergroups.items(): + for g in group: + if g != lower: + lowerpairs.append([g, lower]) + + def printpairs(name, foldpairs): + foldpairs.sort() + foldranges = _MakeRanges(foldpairs) + print("// %d groups, %d pairs, %d ranges" % (len(casegroups), len(foldpairs), len(foldranges))) + print("const CaseFold unicode_%s[] = {" % (name,)) + for lo, hi, delta in foldranges: + print("\t{ %d, %d, %s }," % (lo, hi, delta)) + print("};") + print("const int num_unicode_%s = %d;" % (name, len(foldranges))) + print("") + + print(_header) + printpairs("casefold", foldpairs) + printpairs("tolower", lowerpairs) + print(_trailer) + +if __name__ == '__main__': + main() diff --git a/extern/re2/re2/make_unicode_groups.py b/extern/re2/re2/make_unicode_groups.py new file mode 100644 index 0000000000..46aef40cfb --- /dev/null +++ b/extern/re2/re2/make_unicode_groups.py @@ -0,0 +1,117 @@ +#!/usr/bin/python +# Copyright 2008 The RE2 Authors. All Rights Reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +"""Generate C++ tables for Unicode Script and Category groups.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +import unicode + +_header = """ +// GENERATED BY make_unicode_groups.py; DO NOT EDIT. +// make_unicode_groups.py >unicode_groups.cc + +#include "re2/unicode_groups.h" + +namespace re2 { + +""" + +_trailer = """ + +} // namespace re2 + +""" + +n16 = 0 +n32 = 0 + +def MakeRanges(codes): + """Turn a list like [1,2,3,7,8,9] into a range list [[1,3], [7,9]]""" + ranges = [] + last = -100 + for c in codes: + if c == last+1: + ranges[-1][1] = c + else: + ranges.append([c, c]) + last = c + return ranges + +def PrintRanges(type, name, ranges): + """Print the ranges as an array of type named name.""" + print("static const %s %s[] = {" % (type, name)) + for lo, hi in ranges: + print("\t{ %d, %d }," % (lo, hi)) + print("};") + +# def PrintCodes(type, name, codes): +# """Print the codes as an array of type named name.""" +# print("static %s %s[] = {" % (type, name)) +# for c in codes: +# print("\t%d," % (c,)) +# print("};") + +def PrintGroup(name, codes): + """Print the data structures for the group of codes. + Return a UGroup literal for the group.""" + + # See unicode_groups.h for a description of the data structure. + + # Split codes into 16-bit ranges and 32-bit ranges. + range16 = MakeRanges([c for c in codes if c < 65536]) + range32 = MakeRanges([c for c in codes if c >= 65536]) + + # Pull singleton ranges out of range16. + # code16 = [lo for lo, hi in range16 if lo == hi] + # range16 = [[lo, hi] for lo, hi in range16 if lo != hi] + + global n16 + global n32 + n16 += len(range16) + n32 += len(range32) + + ugroup = "{ \"%s\", +1" % (name,) + # if len(code16) > 0: + # PrintCodes("uint16_t", name+"_code16", code16) + # ugroup += ", %s_code16, %d" % (name, len(code16)) + # else: + # ugroup += ", 0, 0" + if len(range16) > 0: + PrintRanges("URange16", name+"_range16", range16) + ugroup += ", %s_range16, %d" % (name, len(range16)) + else: + ugroup += ", 0, 0" + if len(range32) > 0: + PrintRanges("URange32", name+"_range32", range32) + ugroup += ", %s_range32, %d" % (name, len(range32)) + else: + ugroup += ", 0, 0" + ugroup += " }" + return ugroup + +def main(): + categories = unicode.Categories() + scripts = unicode.Scripts() + print(_header) + ugroups = [] + for name in sorted(categories): + ugroups.append(PrintGroup(name, categories[name])) + for name in sorted(scripts): + ugroups.append(PrintGroup(name, scripts[name])) + print("// %d 16-bit ranges, %d 32-bit ranges" % (n16, n32)) + print("const UGroup unicode_groups[] = {") + ugroups.sort() + for ug in ugroups: + print("\t%s," % (ug,)) + print("};") + print("const int num_unicode_groups = %d;" % (len(ugroups),)) + print(_trailer) + +if __name__ == '__main__': + main() diff --git a/extern/re2/re2/mimics_pcre.cc b/extern/re2/re2/mimics_pcre.cc new file mode 100644 index 0000000000..ad197bef55 --- /dev/null +++ b/extern/re2/re2/mimics_pcre.cc @@ -0,0 +1,187 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Determine whether this library should match PCRE exactly +// for a particular Regexp. (If so, the testing framework can +// check that it does.) +// +// This library matches PCRE except in these cases: +// * the regexp contains a repetition of an empty string, +// like (a*)* or (a*)+. In this case, PCRE will treat +// the repetition sequence as ending with an empty string, +// while this library does not. +// * Perl and PCRE differ on whether \v matches \n. +// For historical reasons, this library implements the Perl behavior. +// * Perl and PCRE allow $ in one-line mode to match either the very +// end of the text or just before a \n at the end of the text. +// This library requires it to match only the end of the text. +// * Similarly, Perl and PCRE do not allow ^ in multi-line mode to +// match the end of the text if the last character is a \n. +// This library does allow it. +// +// Regexp::MimicsPCRE checks for any of these conditions. + +#include "util/util.h" +#include "util/logging.h" +#include "re2/regexp.h" +#include "re2/walker-inl.h" + +namespace re2 { + +// Returns whether re might match an empty string. +static bool CanBeEmptyString(Regexp *re); + +// Walker class to compute whether library handles a regexp +// exactly as PCRE would. See comment at top for conditions. + +class PCREWalker : public Regexp::Walker { + public: + PCREWalker() {} + bool PostVisit(Regexp* re, bool parent_arg, bool pre_arg, bool* child_args, + int nchild_args); + + bool ShortVisit(Regexp* re, bool a) { + // Should never be called: we use Walk not WalkExponential. + LOG(DFATAL) << "EmptyStringWalker::ShortVisit called"; + return a; + } +}; + +// Called after visiting each of re's children and accumulating +// the return values in child_args. So child_args contains whether +// this library mimics PCRE for those subexpressions. +bool PCREWalker::PostVisit(Regexp* re, bool parent_arg, bool pre_arg, + bool* child_args, int nchild_args) { + // If children failed, so do we. + for (int i = 0; i < nchild_args; i++) + if (!child_args[i]) + return false; + + // Otherwise look for other reasons to fail. + switch (re->op()) { + // Look for repeated empty string. + case kRegexpStar: + case kRegexpPlus: + case kRegexpQuest: + if (CanBeEmptyString(re->sub()[0])) + return false; + break; + case kRegexpRepeat: + if (re->max() == -1 && CanBeEmptyString(re->sub()[0])) + return false; + break; + + // Look for \v + case kRegexpLiteral: + if (re->rune() == '\v') + return false; + break; + + // Look for $ in single-line mode. + case kRegexpEndText: + case kRegexpEmptyMatch: + if (re->parse_flags() & Regexp::WasDollar) + return false; + break; + + // Look for ^ in multi-line mode. + case kRegexpBeginLine: + // No condition: in single-line mode ^ becomes kRegexpBeginText. + return false; + + default: + break; + } + + // Not proven guilty. + return true; +} + +// Returns whether this regexp's behavior will mimic PCRE's exactly. +bool Regexp::MimicsPCRE() { + PCREWalker w; + return w.Walk(this, true); +} + + +// Walker class to compute whether a Regexp can match an empty string. +// It is okay to overestimate. For example, \b\B cannot match an empty +// string, because \b and \B are mutually exclusive, but this isn't +// that smart and will say it can. Spurious empty strings +// will reduce the number of regexps we sanity check against PCRE, +// but they won't break anything. + +class EmptyStringWalker : public Regexp::Walker { + public: + EmptyStringWalker() { } + bool PostVisit(Regexp* re, bool parent_arg, bool pre_arg, + bool* child_args, int nchild_args); + + bool ShortVisit(Regexp* re, bool a) { + // Should never be called: we use Walk not WalkExponential. + LOG(DFATAL) << "EmptyStringWalker::ShortVisit called"; + return a; + } + + private: + EmptyStringWalker(const EmptyStringWalker&) = delete; + EmptyStringWalker& operator=(const EmptyStringWalker&) = delete; +}; + +// Called after visiting re's children. child_args contains the return +// value from each of the children's PostVisits (i.e., whether each child +// can match an empty string). Returns whether this clause can match an +// empty string. +bool EmptyStringWalker::PostVisit(Regexp* re, bool parent_arg, bool pre_arg, + bool* child_args, int nchild_args) { + switch (re->op()) { + case kRegexpNoMatch: // never empty + case kRegexpLiteral: + case kRegexpAnyChar: + case kRegexpAnyByte: + case kRegexpCharClass: + case kRegexpLiteralString: + return false; + + case kRegexpEmptyMatch: // always empty + case kRegexpBeginLine: // always empty, when they match + case kRegexpEndLine: + case kRegexpNoWordBoundary: + case kRegexpWordBoundary: + case kRegexpBeginText: + case kRegexpEndText: + case kRegexpStar: // can always be empty + case kRegexpQuest: + case kRegexpHaveMatch: + return true; + + case kRegexpConcat: // can be empty if all children can + for (int i = 0; i < nchild_args; i++) + if (!child_args[i]) + return false; + return true; + + case kRegexpAlternate: // can be empty if any child can + for (int i = 0; i < nchild_args; i++) + if (child_args[i]) + return true; + return false; + + case kRegexpPlus: // can be empty if the child can + case kRegexpCapture: + return child_args[0]; + + case kRegexpRepeat: // can be empty if child can or is x{0} + return child_args[0] || re->min() == 0; + } + return false; +} + +// Returns whether re can match an empty string. +static bool CanBeEmptyString(Regexp* re) { + EmptyStringWalker w; + return w.Walk(re, true); +} + +} // namespace re2 diff --git a/extern/re2/re2/nfa.cc b/extern/re2/re2/nfa.cc new file mode 100644 index 0000000000..7bb4fafae6 --- /dev/null +++ b/extern/re2/re2/nfa.cc @@ -0,0 +1,757 @@ +// Copyright 2006-2007 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tested by search_test.cc. +// +// Prog::SearchNFA, an NFA search. +// This is an actual NFA like the theorists talk about, +// not the pseudo-NFA found in backtracking regexp implementations. +// +// IMPLEMENTATION +// +// This algorithm is a variant of one that appeared in Rob Pike's sam editor, +// which is a variant of the one described in Thompson's 1968 CACM paper. +// See http://swtch.com/~rsc/regexp/ for various history. The main feature +// over the DFA implementation is that it tracks submatch boundaries. +// +// When the choice of submatch boundaries is ambiguous, this particular +// implementation makes the same choices that traditional backtracking +// implementations (in particular, Perl and PCRE) do. +// Note that unlike in Perl and PCRE, this algorithm *cannot* take exponential +// time in the length of the input. +// +// Like Thompson's original machine and like the DFA implementation, this +// implementation notices a match only once it is one byte past it. + +#include +#include +#include +#include +#include +#include + +#include "re2/prog.h" +#include "re2/regexp.h" +#include "util/logging.h" +#include "util/pod_array.h" +#include "util/sparse_array.h" +#include "util/sparse_set.h" +#include "util/strutil.h" + +namespace re2 { + +static const bool ExtraDebug = false; + +class NFA { + public: + NFA(Prog* prog); + ~NFA(); + + // Searches for a matching string. + // * If anchored is true, only considers matches starting at offset. + // Otherwise finds lefmost match at or after offset. + // * If longest is true, returns the longest match starting + // at the chosen start point. Otherwise returns the so-called + // left-biased match, the one traditional backtracking engines + // (like Perl and PCRE) find. + // Records submatch boundaries in submatch[1..nsubmatch-1]. + // Submatch[0] is the entire match. When there is a choice in + // which text matches each subexpression, the submatch boundaries + // are chosen to match what a backtracking implementation would choose. + bool Search(const StringPiece& text, const StringPiece& context, + bool anchored, bool longest, + StringPiece* submatch, int nsubmatch); + + private: + struct Thread { + union { + int ref; + Thread* next; // when on free list + }; + const char** capture; + }; + + // State for explicit stack in AddToThreadq. + struct AddState { + int id; // Inst to process + Thread* t; // if not null, set t0 = t before processing id + }; + + // Threadq is a list of threads. The list is sorted by the order + // in which Perl would explore that particular state -- the earlier + // choices appear earlier in the list. + typedef SparseArray Threadq; + + inline Thread* AllocThread(); + inline Thread* Incref(Thread* t); + inline void Decref(Thread* t); + + // Follows all empty arrows from id0 and enqueues all the states reached. + // Enqueues only the ByteRange instructions that match byte c. + // context is used (with p) for evaluating empty-width specials. + // p is the current input position, and t0 is the current thread. + void AddToThreadq(Threadq* q, int id0, int c, const StringPiece& context, + const char* p, Thread* t0); + + // Run runq on byte c, appending new states to nextq. + // Updates matched_ and match_ as new, better matches are found. + // context is used (with p) for evaluating empty-width specials. + // p is the position of byte c in the input string for AddToThreadq; + // p-1 will be used when processing Match instructions. + // Frees all the threads on runq. + // If there is a shortcut to the end, returns that shortcut. + int Step(Threadq* runq, Threadq* nextq, int c, const StringPiece& context, + const char* p); + + // Returns text version of capture information, for debugging. + std::string FormatCapture(const char** capture); + + inline void CopyCapture(const char** dst, const char** src); + + Prog* prog_; // underlying program + int start_; // start instruction in program + int ncapture_; // number of submatches to track + bool longest_; // whether searching for longest match + bool endmatch_; // whether match must end at text.end() + const char* btext_; // beginning of text being matched (for FormatSubmatch) + const char* etext_; // end of text being matched (for endmatch_) + Threadq q0_, q1_; // pre-allocated for Search. + PODArray stack_; // pre-allocated for AddToThreadq + Thread* free_threads_; // free list + const char** match_; // best match so far + bool matched_; // any match so far? + + NFA(const NFA&) = delete; + NFA& operator=(const NFA&) = delete; +}; + +NFA::NFA(Prog* prog) { + prog_ = prog; + start_ = prog_->start(); + ncapture_ = 0; + longest_ = false; + endmatch_ = false; + btext_ = NULL; + etext_ = NULL; + q0_.resize(prog_->size()); + q1_.resize(prog_->size()); + // See NFA::AddToThreadq() for why this is so. + int nstack = 2*prog_->inst_count(kInstCapture) + + prog_->inst_count(kInstEmptyWidth) + + prog_->inst_count(kInstNop) + 1; // + 1 for start inst + stack_ = PODArray(nstack); + free_threads_ = NULL; + match_ = NULL; + matched_ = false; +} + +NFA::~NFA() { + delete[] match_; + Thread* next; + for (Thread* t = free_threads_; t; t = next) { + next = t->next; + delete[] t->capture; + delete t; + } +} + +NFA::Thread* NFA::AllocThread() { + Thread* t = free_threads_; + if (t == NULL) { + t = new Thread; + t->ref = 1; + t->capture = new const char*[ncapture_]; + return t; + } + free_threads_ = t->next; + t->ref = 1; + return t; +} + +NFA::Thread* NFA::Incref(Thread* t) { + DCHECK(t != NULL); + t->ref++; + return t; +} + +void NFA::Decref(Thread* t) { + if (t == NULL) + return; + t->ref--; + if (t->ref > 0) + return; + DCHECK_EQ(t->ref, 0); + t->next = free_threads_; + free_threads_ = t; +} + +void NFA::CopyCapture(const char** dst, const char** src) { + for (int i = 0; i < ncapture_; i+=2) { + dst[i] = src[i]; + dst[i+1] = src[i+1]; + } +} + +// Follows all empty arrows from id0 and enqueues all the states reached. +// Enqueues only the ByteRange instructions that match byte c. +// context is used (with p) for evaluating empty-width specials. +// p is the current input position, and t0 is the current thread. +void NFA::AddToThreadq(Threadq* q, int id0, int c, const StringPiece& context, + const char* p, Thread* t0) { + if (id0 == 0) + return; + + // Use stack_ to hold our stack of instructions yet to process. + // It was preallocated as follows: + // two entries per Capture; + // one entry per EmptyWidth; and + // one entry per Nop. + // This reflects the maximum number of stack pushes that each can + // perform. (Each instruction can be processed at most once.) + AddState* stk = stack_.data(); + int nstk = 0; + + stk[nstk++] = {id0, NULL}; + while (nstk > 0) { + DCHECK_LE(nstk, stack_.size()); + AddState a = stk[--nstk]; + + Loop: + if (a.t != NULL) { + // t0 was a thread that we allocated and copied in order to + // record the capture, so we must now decref it. + Decref(t0); + t0 = a.t; + } + + int id = a.id; + if (id == 0) + continue; + if (q->has_index(id)) { + if (ExtraDebug) + fprintf(stderr, " [%d%s]\n", id, FormatCapture(t0->capture).c_str()); + continue; + } + + // Create entry in q no matter what. We might fill it in below, + // or we might not. Even if not, it is necessary to have it, + // so that we don't revisit id0 during the recursion. + q->set_new(id, NULL); + Thread** tp = &q->get_existing(id); + int j; + Thread* t; + Prog::Inst* ip = prog_->inst(id); + switch (ip->opcode()) { + default: + LOG(DFATAL) << "unhandled " << ip->opcode() << " in AddToThreadq"; + break; + + case kInstFail: + break; + + case kInstAltMatch: + // Save state; will pick up at next byte. + t = Incref(t0); + *tp = t; + + DCHECK(!ip->last()); + a = {id+1, NULL}; + goto Loop; + + case kInstNop: + if (!ip->last()) + stk[nstk++] = {id+1, NULL}; + + // Continue on. + a = {ip->out(), NULL}; + goto Loop; + + case kInstCapture: + if (!ip->last()) + stk[nstk++] = {id+1, NULL}; + + if ((j=ip->cap()) < ncapture_) { + // Push a dummy whose only job is to restore t0 + // once we finish exploring this possibility. + stk[nstk++] = {0, t0}; + + // Record capture. + t = AllocThread(); + CopyCapture(t->capture, t0->capture); + t->capture[j] = p; + t0 = t; + } + a = {ip->out(), NULL}; + goto Loop; + + case kInstByteRange: + if (!ip->Matches(c)) + goto Next; + + // Save state; will pick up at next byte. + t = Incref(t0); + *tp = t; + if (ExtraDebug) + fprintf(stderr, " + %d%s\n", id, FormatCapture(t0->capture).c_str()); + + if (ip->hint() == 0) + break; + a = {id+ip->hint(), NULL}; + goto Loop; + + case kInstMatch: + // Save state; will pick up at next byte. + t = Incref(t0); + *tp = t; + if (ExtraDebug) + fprintf(stderr, " ! %d%s\n", id, FormatCapture(t0->capture).c_str()); + + Next: + if (ip->last()) + break; + a = {id+1, NULL}; + goto Loop; + + case kInstEmptyWidth: + if (!ip->last()) + stk[nstk++] = {id+1, NULL}; + + // Continue on if we have all the right flag bits. + if (ip->empty() & ~Prog::EmptyFlags(context, p)) + break; + a = {ip->out(), NULL}; + goto Loop; + } + } +} + +// Run runq on byte c, appending new states to nextq. +// Updates matched_ and match_ as new, better matches are found. +// context is used (with p) for evaluating empty-width specials. +// p is the position of byte c in the input string for AddToThreadq; +// p-1 will be used when processing Match instructions. +// Frees all the threads on runq. +// If there is a shortcut to the end, returns that shortcut. +int NFA::Step(Threadq* runq, Threadq* nextq, int c, const StringPiece& context, + const char* p) { + nextq->clear(); + + for (Threadq::iterator i = runq->begin(); i != runq->end(); ++i) { + Thread* t = i->value(); + if (t == NULL) + continue; + + if (longest_) { + // Can skip any threads started after our current best match. + if (matched_ && match_[0] < t->capture[0]) { + Decref(t); + continue; + } + } + + int id = i->index(); + Prog::Inst* ip = prog_->inst(id); + + switch (ip->opcode()) { + default: + // Should only see the values handled below. + LOG(DFATAL) << "Unhandled " << ip->opcode() << " in step"; + break; + + case kInstByteRange: + AddToThreadq(nextq, ip->out(), c, context, p, t); + break; + + case kInstAltMatch: + if (i != runq->begin()) + break; + // The match is ours if we want it. + if (ip->greedy(prog_) || longest_) { + CopyCapture(match_, t->capture); + matched_ = true; + + Decref(t); + for (++i; i != runq->end(); ++i) + Decref(i->value()); + runq->clear(); + if (ip->greedy(prog_)) + return ip->out1(); + return ip->out(); + } + break; + + case kInstMatch: { + // Avoid invoking undefined behavior when p happens + // to be null - and p-1 would be meaningless anyway. + if (p == NULL) + break; + + if (endmatch_ && p-1 != etext_) + break; + + if (longest_) { + // Leftmost-longest mode: save this match only if + // it is either farther to the left or at the same + // point but longer than an existing match. + if (!matched_ || t->capture[0] < match_[0] || + (t->capture[0] == match_[0] && p-1 > match_[1])) { + CopyCapture(match_, t->capture); + match_[1] = p-1; + matched_ = true; + } + } else { + // Leftmost-biased mode: this match is by definition + // better than what we've already found (see next line). + CopyCapture(match_, t->capture); + match_[1] = p-1; + matched_ = true; + + // Cut off the threads that can only find matches + // worse than the one we just found: don't run the + // rest of the current Threadq. + Decref(t); + for (++i; i != runq->end(); ++i) + Decref(i->value()); + runq->clear(); + return 0; + } + break; + } + } + Decref(t); + } + runq->clear(); + return 0; +} + +std::string NFA::FormatCapture(const char** capture) { + std::string s; + for (int i = 0; i < ncapture_; i+=2) { + if (capture[i] == NULL) + s += "(?,?)"; + else if (capture[i+1] == NULL) + s += StringPrintf("(%d,?)", + (int)(capture[i] - btext_)); + else + s += StringPrintf("(%d,%d)", + (int)(capture[i] - btext_), + (int)(capture[i+1] - btext_)); + } + return s; +} + +bool NFA::Search(const StringPiece& text, const StringPiece& const_context, + bool anchored, bool longest, + StringPiece* submatch, int nsubmatch) { + if (start_ == 0) + return false; + + StringPiece context = const_context; + if (context.begin() == NULL) + context = text; + + // Sanity check: make sure that text lies within context. + if (text.begin() < context.begin() || text.end() > context.end()) { + LOG(DFATAL) << "context does not contain text"; + return false; + } + + if (prog_->anchor_start() && context.begin() != text.begin()) + return false; + if (prog_->anchor_end() && context.end() != text.end()) + return false; + anchored |= prog_->anchor_start(); + if (prog_->anchor_end()) { + longest = true; + endmatch_ = true; + etext_ = text.end(); + } + + if (nsubmatch < 0) { + LOG(DFATAL) << "Bad args: nsubmatch=" << nsubmatch; + return false; + } + + // Save search parameters. + ncapture_ = 2*nsubmatch; + longest_ = longest; + + if (nsubmatch == 0) { + // We need to maintain match[0], both to distinguish the + // longest match (if longest is true) and also to tell + // whether we've seen any matches at all. + ncapture_ = 2; + } + + match_ = new const char*[ncapture_]; + matched_ = false; + + // For debugging prints. + btext_ = context.begin(); + + if (ExtraDebug) + fprintf(stderr, "NFA::Search %s (context: %s) anchored=%d longest=%d\n", + std::string(text).c_str(), std::string(context).c_str(), anchored, + longest); + + // Set up search. + Threadq* runq = &q0_; + Threadq* nextq = &q1_; + runq->clear(); + nextq->clear(); + memset(&match_[0], 0, ncapture_*sizeof match_[0]); + + // Loop over the text, stepping the machine. + for (const char* p = text.begin();; p++) { + if (ExtraDebug) { + int c = 0; + if (p == context.begin()) + c = '^'; + else if (p > text.end()) + c = '$'; + else if (p < text.end()) + c = p[0] & 0xFF; + + fprintf(stderr, "%c:", c); + for (Threadq::iterator i = runq->begin(); i != runq->end(); ++i) { + Thread* t = i->value(); + if (t == NULL) + continue; + fprintf(stderr, " %d%s", i->index(), FormatCapture(t->capture).c_str()); + } + fprintf(stderr, "\n"); + } + + // This is a no-op the first time around the loop because runq is empty. + int id = Step(runq, nextq, p < text.end() ? p[0] & 0xFF : -1, context, p); + DCHECK_EQ(runq->size(), 0); + using std::swap; + swap(nextq, runq); + nextq->clear(); + if (id != 0) { + // We're done: full match ahead. + p = text.end(); + for (;;) { + Prog::Inst* ip = prog_->inst(id); + switch (ip->opcode()) { + default: + LOG(DFATAL) << "Unexpected opcode in short circuit: " << ip->opcode(); + break; + + case kInstCapture: + if (ip->cap() < ncapture_) + match_[ip->cap()] = p; + id = ip->out(); + continue; + + case kInstNop: + id = ip->out(); + continue; + + case kInstMatch: + match_[1] = p; + matched_ = true; + break; + } + break; + } + break; + } + + if (p > text.end()) + break; + + // Start a new thread if there have not been any matches. + // (No point in starting a new thread if there have been + // matches, since it would be to the right of the match + // we already found.) + if (!matched_ && (!anchored || p == text.begin())) { + // If there's a required first byte for an unanchored search + // and we're not in the middle of any possible matches, + // use memchr to search for the byte quickly. + int fb = prog_->first_byte(); + if (!anchored && runq->size() == 0 && + fb >= 0 && p < text.end() && (p[0] & 0xFF) != fb) { + p = reinterpret_cast(memchr(p, fb, text.end() - p)); + if (p == NULL) { + p = text.end(); + } + } + + Thread* t = AllocThread(); + CopyCapture(t->capture, match_); + t->capture[0] = p; + AddToThreadq(runq, start_, p < text.end() ? p[0] & 0xFF : -1, context, p, + t); + Decref(t); + } + + // If all the threads have died, stop early. + if (runq->size() == 0) { + if (ExtraDebug) + fprintf(stderr, "dead\n"); + break; + } + } + + for (Threadq::iterator i = runq->begin(); i != runq->end(); ++i) + Decref(i->value()); + + if (matched_) { + for (int i = 0; i < nsubmatch; i++) + submatch[i] = + StringPiece(match_[2 * i], + static_cast(match_[2 * i + 1] - match_[2 * i])); + if (ExtraDebug) + fprintf(stderr, "match (%td,%td)\n", + match_[0] - btext_, match_[1] - btext_); + return true; + } + return false; +} + +// Computes whether all successful matches have a common first byte, +// and if so, returns that byte. If not, returns -1. +int Prog::ComputeFirstByte() { + int b = -1; + SparseSet q(size()); + q.insert(start()); + for (SparseSet::iterator it = q.begin(); it != q.end(); ++it) { + int id = *it; + Prog::Inst* ip = inst(id); + switch (ip->opcode()) { + default: + LOG(DFATAL) << "unhandled " << ip->opcode() << " in ComputeFirstByte"; + break; + + case kInstMatch: + // The empty string matches: no first byte. + return -1; + + case kInstByteRange: + if (!ip->last()) + q.insert(id+1); + + // Must match only a single byte + if (ip->lo() != ip->hi()) + return -1; + if (ip->foldcase() && 'a' <= ip->lo() && ip->lo() <= 'z') + return -1; + // If we haven't seen any bytes yet, record it; + // otherwise must match the one we saw before. + if (b == -1) + b = ip->lo(); + else if (b != ip->lo()) + return -1; + break; + + case kInstNop: + case kInstCapture: + case kInstEmptyWidth: + if (!ip->last()) + q.insert(id+1); + + // Continue on. + // Ignore ip->empty() flags for kInstEmptyWidth + // in order to be as conservative as possible + // (assume all possible empty-width flags are true). + if (ip->out()) + q.insert(ip->out()); + break; + + case kInstAltMatch: + DCHECK(!ip->last()); + q.insert(id+1); + break; + + case kInstFail: + break; + } + } + return b; +} + +bool +Prog::SearchNFA(const StringPiece& text, const StringPiece& context, + Anchor anchor, MatchKind kind, + StringPiece* match, int nmatch) { + if (ExtraDebug) + Dump(); + + NFA nfa(this); + StringPiece sp; + if (kind == kFullMatch) { + anchor = kAnchored; + if (nmatch == 0) { + match = &sp; + nmatch = 1; + } + } + if (!nfa.Search(text, context, anchor == kAnchored, kind != kFirstMatch, match, nmatch)) + return false; + if (kind == kFullMatch && match[0].end() != text.end()) + return false; + return true; +} + +// For each instruction i in the program reachable from the start, compute the +// number of instructions reachable from i by following only empty transitions +// and record that count as fanout[i]. +// +// fanout holds the results and is also the work queue for the outer iteration. +// reachable holds the reached nodes for the inner iteration. +void Prog::Fanout(SparseArray* fanout) { + DCHECK_EQ(fanout->max_size(), size()); + SparseSet reachable(size()); + fanout->clear(); + fanout->set_new(start(), 0); + for (SparseArray::iterator i = fanout->begin(); i != fanout->end(); ++i) { + int* count = &i->value(); + reachable.clear(); + reachable.insert(i->index()); + for (SparseSet::iterator j = reachable.begin(); j != reachable.end(); ++j) { + int id = *j; + Prog::Inst* ip = inst(id); + switch (ip->opcode()) { + default: + LOG(DFATAL) << "unhandled " << ip->opcode() << " in Prog::Fanout()"; + break; + + case kInstByteRange: + if (!ip->last()) + reachable.insert(id+1); + + (*count)++; + if (!fanout->has_index(ip->out())) { + fanout->set_new(ip->out(), 0); + } + break; + + case kInstAltMatch: + DCHECK(!ip->last()); + reachable.insert(id+1); + break; + + case kInstCapture: + case kInstEmptyWidth: + case kInstNop: + if (!ip->last()) + reachable.insert(id+1); + + reachable.insert(ip->out()); + break; + + case kInstMatch: + if (!ip->last()) + reachable.insert(id+1); + break; + + case kInstFail: + break; + } + } + } +} + +} // namespace re2 diff --git a/extern/re2/re2/onepass.cc b/extern/re2/re2/onepass.cc new file mode 100644 index 0000000000..d615893274 --- /dev/null +++ b/extern/re2/re2/onepass.cc @@ -0,0 +1,623 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tested by search_test.cc. +// +// Prog::SearchOnePass is an efficient implementation of +// regular expression search with submatch tracking for +// what I call "one-pass regular expressions". (An alternate +// name might be "backtracking-free regular expressions".) +// +// One-pass regular expressions have the property that +// at each input byte during an anchored match, there may be +// multiple alternatives but only one can proceed for any +// given input byte. +// +// For example, the regexp /x*yx*/ is one-pass: you read +// x's until a y, then you read the y, then you keep reading x's. +// At no point do you have to guess what to do or back up +// and try a different guess. +// +// On the other hand, /x*x/ is not one-pass: when you're +// looking at an input "x", it's not clear whether you should +// use it to extend the x* or as the final x. +// +// More examples: /([^ ]*) (.*)/ is one-pass; /(.*) (.*)/ is not. +// /(\d+)-(\d+)/ is one-pass; /(\d+).(\d+)/ is not. +// +// A simple intuition for identifying one-pass regular expressions +// is that it's always immediately obvious when a repetition ends. +// It must also be immediately obvious which branch of an | to take: +// +// /x(y|z)/ is one-pass, but /(xy|xz)/ is not. +// +// The NFA-based search in nfa.cc does some bookkeeping to +// avoid the need for backtracking and its associated exponential blowup. +// But if we have a one-pass regular expression, there is no +// possibility of backtracking, so there is no need for the +// extra bookkeeping. Hence, this code. +// +// On a one-pass regular expression, the NFA code in nfa.cc +// runs at about 1/20 of the backtracking-based PCRE speed. +// In contrast, the code in this file runs at about the same +// speed as PCRE. +// +// One-pass regular expressions get used a lot when RE is +// used for parsing simple strings, so it pays off to +// notice them and handle them efficiently. +// +// See also Anne Brüggemann-Klein and Derick Wood, +// "One-unambiguous regular languages", Information and Computation 142(2). + +#include +#include +#include +#include +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "util/pod_array.h" +#include "util/sparse_set.h" +#include "util/strutil.h" +#include "util/utf.h" +#include "re2/prog.h" +#include "re2/stringpiece.h" + +// Silence "zero-sized array in struct/union" warning for OneState::action. +#ifdef _MSC_VER +#pragma warning(disable: 4200) +#endif + +namespace re2 { + +static const bool ExtraDebug = false; + +// The key insight behind this implementation is that the +// non-determinism in an NFA for a one-pass regular expression +// is contained. To explain what that means, first a +// refresher about what regular expression programs look like +// and how the usual NFA execution runs. +// +// In a regular expression program, only the kInstByteRange +// instruction processes an input byte c and moves on to the +// next byte in the string (it does so if c is in the given range). +// The kInstByteRange instructions correspond to literal characters +// and character classes in the regular expression. +// +// The kInstAlt instructions are used as wiring to connect the +// kInstByteRange instructions together in interesting ways when +// implementing | + and *. +// The kInstAlt instruction forks execution, like a goto that +// jumps to ip->out() and ip->out1() in parallel. Each of the +// resulting computation paths is called a thread. +// +// The other instructions -- kInstEmptyWidth, kInstMatch, kInstCapture -- +// are interesting in their own right but like kInstAlt they don't +// advance the input pointer. Only kInstByteRange does. +// +// The automaton execution in nfa.cc runs all the possible +// threads of execution in lock-step over the input. To process +// a particular byte, each thread gets run until it either dies +// or finds a kInstByteRange instruction matching the byte. +// If the latter happens, the thread stops just past the +// kInstByteRange instruction (at ip->out()) and waits for +// the other threads to finish processing the input byte. +// Then, once all the threads have processed that input byte, +// the whole process repeats. The kInstAlt state instruction +// might create new threads during input processing, but no +// matter what, all the threads stop after a kInstByteRange +// and wait for the other threads to "catch up". +// Running in lock step like this ensures that the NFA reads +// the input string only once. +// +// Each thread maintains its own set of capture registers +// (the string positions at which it executed the kInstCapture +// instructions corresponding to capturing parentheses in the +// regular expression). Repeated copying of the capture registers +// is the main performance bottleneck in the NFA implementation. +// +// A regular expression program is "one-pass" if, no matter what +// the input string, there is only one thread that makes it +// past a kInstByteRange instruction at each input byte. This means +// that there is in some sense only one active thread throughout +// the execution. Other threads might be created during the +// processing of an input byte, but they are ephemeral: only one +// thread is left to start processing the next input byte. +// This is what I meant above when I said the non-determinism +// was "contained". +// +// To execute a one-pass regular expression program, we can build +// a DFA (no non-determinism) that has at most as many states as +// the NFA (compare this to the possibly exponential number of states +// in the general case). Each state records, for each possible +// input byte, the next state along with the conditions required +// before entering that state -- empty-width flags that must be true +// and capture operations that must be performed. It also records +// whether a set of conditions required to finish a match at that +// point in the input rather than process the next byte. + +// A state in the one-pass NFA - just an array of actions indexed +// by the bytemap_[] of the next input byte. (The bytemap +// maps next input bytes into equivalence classes, to reduce +// the memory footprint.) +struct OneState { + uint32_t matchcond; // conditions to match right now. + uint32_t action[]; +}; + +// The uint32_t conditions in the action are a combination of +// condition and capture bits and the next state. The bottom 16 bits +// are the condition and capture bits, and the top 16 are the index of +// the next state. +// +// Bits 0-5 are the empty-width flags from prog.h. +// Bit 6 is kMatchWins, which means the match takes +// priority over moving to next in a first-match search. +// The remaining bits mark capture registers that should +// be set to the current input position. The capture bits +// start at index 2, since the search loop can take care of +// cap[0], cap[1] (the overall match position). +// That means we can handle up to 5 capturing parens: $1 through $4, plus $0. +// No input position can satisfy both kEmptyWordBoundary +// and kEmptyNonWordBoundary, so we can use that as a sentinel +// instead of needing an extra bit. + +static const int kIndexShift = 16; // number of bits below index +static const int kEmptyShift = 6; // number of empty flags in prog.h +static const int kRealCapShift = kEmptyShift + 1; +static const int kRealMaxCap = (kIndexShift - kRealCapShift) / 2 * 2; + +// Parameters used to skip over cap[0], cap[1]. +static const int kCapShift = kRealCapShift - 2; +static const int kMaxCap = kRealMaxCap + 2; + +static const uint32_t kMatchWins = 1 << kEmptyShift; +static const uint32_t kCapMask = ((1 << kRealMaxCap) - 1) << kRealCapShift; + +static const uint32_t kImpossible = kEmptyWordBoundary | kEmptyNonWordBoundary; + +// Check, at compile time, that prog.h agrees with math above. +// This function is never called. +void OnePass_Checks() { + static_assert((1<(nodes + statesize*nodeindex); +} + +bool Prog::SearchOnePass(const StringPiece& text, + const StringPiece& const_context, + Anchor anchor, MatchKind kind, + StringPiece* match, int nmatch) { + if (anchor != kAnchored && kind != kFullMatch) { + LOG(DFATAL) << "Cannot use SearchOnePass for unanchored matches."; + return false; + } + + // Make sure we have at least cap[1], + // because we use it to tell if we matched. + int ncap = 2*nmatch; + if (ncap < 2) + ncap = 2; + + const char* cap[kMaxCap]; + for (int i = 0; i < ncap; i++) + cap[i] = NULL; + + const char* matchcap[kMaxCap]; + for (int i = 0; i < ncap; i++) + matchcap[i] = NULL; + + StringPiece context = const_context; + if (context.begin() == NULL) + context = text; + if (anchor_start() && context.begin() != text.begin()) + return false; + if (anchor_end() && context.end() != text.end()) + return false; + if (anchor_end()) + kind = kFullMatch; + + uint8_t* nodes = onepass_nodes_.data(); + int statesize = sizeof(OneState) + bytemap_range()*sizeof(uint32_t); + // start() is always mapped to the zeroth OneState. + OneState* state = IndexToNode(nodes, statesize, 0); + uint8_t* bytemap = bytemap_; + const char* bp = text.begin(); + const char* ep = text.end(); + const char* p; + bool matched = false; + matchcap[0] = bp; + cap[0] = bp; + uint32_t nextmatchcond = state->matchcond; + for (p = bp; p < ep; p++) { + int c = bytemap[*p & 0xFF]; + uint32_t matchcond = nextmatchcond; + uint32_t cond = state->action[c]; + + // Determine whether we can reach act->next. + // If so, advance state and nextmatchcond. + if ((cond & kEmptyAllFlags) == 0 || Satisfy(cond, context, p)) { + uint32_t nextindex = cond >> kIndexShift; + state = IndexToNode(nodes, statesize, nextindex); + nextmatchcond = state->matchcond; + } else { + state = NULL; + nextmatchcond = kImpossible; + } + + // This code section is carefully tuned. + // The goto sequence is about 10% faster than the + // obvious rewrite as a large if statement in the + // ASCIIMatchRE2 and DotMatchRE2 benchmarks. + + // Saving the match capture registers is expensive. + // Is this intermediate match worth thinking about? + + // Not if we want a full match. + if (kind == kFullMatch) + goto skipmatch; + + // Not if it's impossible. + if (matchcond == kImpossible) + goto skipmatch; + + // Not if the possible match is beaten by the certain + // match at the next byte. When this test is useless + // (e.g., HTTPPartialMatchRE2) it slows the loop by + // about 10%, but when it avoids work (e.g., DotMatchRE2), + // it cuts the loop execution by about 45%. + if ((cond & kMatchWins) == 0 && (nextmatchcond & kEmptyAllFlags) == 0) + goto skipmatch; + + // Finally, the match conditions must be satisfied. + if ((matchcond & kEmptyAllFlags) == 0 || Satisfy(matchcond, context, p)) { + for (int i = 2; i < 2*nmatch; i++) + matchcap[i] = cap[i]; + if (nmatch > 1 && (matchcond & kCapMask)) + ApplyCaptures(matchcond, p, matchcap, ncap); + matchcap[1] = p; + matched = true; + + // If we're in longest match mode, we have to keep + // going and see if we find a longer match. + // In first match mode, we can stop if the match + // takes priority over the next state for this input byte. + // That bit is per-input byte and thus in cond, not matchcond. + if (kind == kFirstMatch && (cond & kMatchWins)) + goto done; + } + + skipmatch: + if (state == NULL) + goto done; + if ((cond & kCapMask) && nmatch > 1) + ApplyCaptures(cond, p, cap, ncap); + } + + // Look for match at end of input. + { + uint32_t matchcond = state->matchcond; + if (matchcond != kImpossible && + ((matchcond & kEmptyAllFlags) == 0 || Satisfy(matchcond, context, p))) { + if (nmatch > 1 && (matchcond & kCapMask)) + ApplyCaptures(matchcond, p, cap, ncap); + for (int i = 2; i < ncap; i++) + matchcap[i] = cap[i]; + matchcap[1] = p; + matched = true; + } + } + +done: + if (!matched) + return false; + for (int i = 0; i < nmatch; i++) + match[i] = + StringPiece(matchcap[2 * i], + static_cast(matchcap[2 * i + 1] - matchcap[2 * i])); + return true; +} + + +// Analysis to determine whether a given regexp program is one-pass. + +// If ip is not on workq, adds ip to work queue and returns true. +// If ip is already on work queue, does nothing and returns false. +// If ip is NULL, does nothing and returns true (pretends to add it). +typedef SparseSet Instq; +static bool AddQ(Instq *q, int id) { + if (id == 0) + return true; + if (q->contains(id)) + return false; + q->insert(id); + return true; +} + +struct InstCond { + int id; + uint32_t cond; +}; + +// Returns whether this is a one-pass program; that is, +// returns whether it is safe to use SearchOnePass on this program. +// These conditions must be true for any instruction ip: +// +// (1) for any other Inst nip, there is at most one input-free +// path from ip to nip. +// (2) there is at most one kInstByte instruction reachable from +// ip that matches any particular byte c. +// (3) there is at most one input-free path from ip to a kInstMatch +// instruction. +// +// This is actually just a conservative approximation: it might +// return false when the answer is true, when kInstEmptyWidth +// instructions are involved. +// Constructs and saves corresponding one-pass NFA on success. +bool Prog::IsOnePass() { + if (did_onepass_) + return onepass_nodes_.data() != NULL; + did_onepass_ = true; + + if (start() == 0) // no match + return false; + + // Steal memory for the one-pass NFA from the overall DFA budget. + // Willing to use at most 1/4 of the DFA budget (heuristic). + // Limit max node count to 65000 as a conservative estimate to + // avoid overflowing 16-bit node index in encoding. + int maxnodes = 2 + inst_count(kInstByteRange); + int statesize = sizeof(OneState) + bytemap_range()*sizeof(uint32_t); + if (maxnodes >= 65000 || dfa_mem_ / 4 / statesize < maxnodes) + return false; + + // Flood the graph starting at the start state, and check + // that in each reachable state, each possible byte leads + // to a unique next state. + int stacksize = inst_count(kInstCapture) + + inst_count(kInstEmptyWidth) + + inst_count(kInstNop) + 1; // + 1 for start inst + PODArray stack(stacksize); + + int size = this->size(); + PODArray nodebyid(size); // indexed by ip + memset(nodebyid.data(), 0xFF, size*sizeof nodebyid[0]); + + // Originally, nodes was a uint8_t[maxnodes*statesize], but that was + // unnecessarily optimistic: why allocate a large amount of memory + // upfront for a large program when it is unlikely to be one-pass? + std::vector nodes; + + Instq tovisit(size), workq(size); + AddQ(&tovisit, start()); + nodebyid[start()] = 0; + int nalloc = 1; + nodes.insert(nodes.end(), statesize, 0); + for (Instq::iterator it = tovisit.begin(); it != tovisit.end(); ++it) { + int id = *it; + int nodeindex = nodebyid[id]; + OneState* node = IndexToNode(nodes.data(), statesize, nodeindex); + + // Flood graph using manual stack, filling in actions as found. + // Default is none. + for (int b = 0; b < bytemap_range_; b++) + node->action[b] = kImpossible; + node->matchcond = kImpossible; + + workq.clear(); + bool matched = false; + int nstack = 0; + stack[nstack].id = id; + stack[nstack++].cond = 0; + while (nstack > 0) { + int id = stack[--nstack].id; + uint32_t cond = stack[nstack].cond; + + Loop: + Prog::Inst* ip = inst(id); + switch (ip->opcode()) { + default: + LOG(DFATAL) << "unhandled opcode: " << ip->opcode(); + break; + + case kInstAltMatch: + // TODO(rsc): Ignoring kInstAltMatch optimization. + // Should implement it in this engine, but it's subtle. + DCHECK(!ip->last()); + // If already on work queue, (1) is violated: bail out. + if (!AddQ(&workq, id+1)) + goto fail; + id = id+1; + goto Loop; + + case kInstByteRange: { + int nextindex = nodebyid[ip->out()]; + if (nextindex == -1) { + if (nalloc >= maxnodes) { + if (ExtraDebug) + LOG(ERROR) << StringPrintf( + "Not OnePass: hit node limit %d >= %d", nalloc, maxnodes); + goto fail; + } + nextindex = nalloc; + AddQ(&tovisit, ip->out()); + nodebyid[ip->out()] = nalloc; + nalloc++; + nodes.insert(nodes.end(), statesize, 0); + // Update node because it might have been invalidated. + node = IndexToNode(nodes.data(), statesize, nodeindex); + } + for (int c = ip->lo(); c <= ip->hi(); c++) { + int b = bytemap_[c]; + // Skip any bytes immediately after c that are also in b. + while (c < 256-1 && bytemap_[c+1] == b) + c++; + uint32_t act = node->action[b]; + uint32_t newact = (nextindex << kIndexShift) | cond; + if (matched) + newact |= kMatchWins; + if ((act & kImpossible) == kImpossible) { + node->action[b] = newact; + } else if (act != newact) { + if (ExtraDebug) + LOG(ERROR) << StringPrintf( + "Not OnePass: conflict on byte %#x at state %d", c, *it); + goto fail; + } + } + if (ip->foldcase()) { + Rune lo = std::max(ip->lo(), 'a') + 'A' - 'a'; + Rune hi = std::min(ip->hi(), 'z') + 'A' - 'a'; + for (int c = lo; c <= hi; c++) { + int b = bytemap_[c]; + // Skip any bytes immediately after c that are also in b. + while (c < 256-1 && bytemap_[c+1] == b) + c++; + uint32_t act = node->action[b]; + uint32_t newact = (nextindex << kIndexShift) | cond; + if (matched) + newact |= kMatchWins; + if ((act & kImpossible) == kImpossible) { + node->action[b] = newact; + } else if (act != newact) { + if (ExtraDebug) + LOG(ERROR) << StringPrintf( + "Not OnePass: conflict on byte %#x at state %d", c, *it); + goto fail; + } + } + } + + if (ip->last()) + break; + // If already on work queue, (1) is violated: bail out. + if (!AddQ(&workq, id+1)) + goto fail; + id = id+1; + goto Loop; + } + + case kInstCapture: + case kInstEmptyWidth: + case kInstNop: + if (!ip->last()) { + // If already on work queue, (1) is violated: bail out. + if (!AddQ(&workq, id+1)) + goto fail; + stack[nstack].id = id+1; + stack[nstack++].cond = cond; + } + + if (ip->opcode() == kInstCapture && ip->cap() < kMaxCap) + cond |= (1 << kCapShift) << ip->cap(); + if (ip->opcode() == kInstEmptyWidth) + cond |= ip->empty(); + + // kInstCapture and kInstNop always proceed to ip->out(). + // kInstEmptyWidth only sometimes proceeds to ip->out(), + // but as a conservative approximation we assume it always does. + // We could be a little more precise by looking at what c + // is, but that seems like overkill. + + // If already on work queue, (1) is violated: bail out. + if (!AddQ(&workq, ip->out())) { + if (ExtraDebug) + LOG(ERROR) << StringPrintf( + "Not OnePass: multiple paths %d -> %d\n", *it, ip->out()); + goto fail; + } + id = ip->out(); + goto Loop; + + case kInstMatch: + if (matched) { + // (3) is violated + if (ExtraDebug) + LOG(ERROR) << StringPrintf( + "Not OnePass: multiple matches from %d\n", *it); + goto fail; + } + matched = true; + node->matchcond = cond; + + if (ip->last()) + break; + // If already on work queue, (1) is violated: bail out. + if (!AddQ(&workq, id+1)) + goto fail; + id = id+1; + goto Loop; + + case kInstFail: + break; + } + } + } + + if (ExtraDebug) { // For debugging, dump one-pass NFA to LOG(ERROR). + LOG(ERROR) << "bytemap:\n" << DumpByteMap(); + LOG(ERROR) << "prog:\n" << Dump(); + + std::map idmap; + for (int i = 0; i < size; i++) + if (nodebyid[i] != -1) + idmap[nodebyid[i]] = i; + + std::string dump; + for (Instq::iterator it = tovisit.begin(); it != tovisit.end(); ++it) { + int id = *it; + int nodeindex = nodebyid[id]; + if (nodeindex == -1) + continue; + OneState* node = IndexToNode(nodes.data(), statesize, nodeindex); + dump += StringPrintf("node %d id=%d: matchcond=%#x\n", + nodeindex, id, node->matchcond); + for (int i = 0; i < bytemap_range_; i++) { + if ((node->action[i] & kImpossible) == kImpossible) + continue; + dump += StringPrintf(" %d cond %#x -> %d id=%d\n", + i, node->action[i] & 0xFFFF, + node->action[i] >> kIndexShift, + idmap[node->action[i] >> kIndexShift]); + } + } + LOG(ERROR) << "nodes:\n" << dump; + } + + dfa_mem_ -= nalloc*statesize; + onepass_nodes_ = PODArray(nalloc*statesize); + memmove(onepass_nodes_.data(), nodes.data(), nalloc*statesize); + return true; + +fail: + return false; +} + +} // namespace re2 diff --git a/extern/re2/re2/parse.cc b/extern/re2/re2/parse.cc new file mode 100644 index 0000000000..93b922a146 --- /dev/null +++ b/extern/re2/re2/parse.cc @@ -0,0 +1,2463 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Regular expression parser. + +// The parser is a simple precedence-based parser with a +// manual stack. The parsing work is done by the methods +// of the ParseState class. The Regexp::Parse function is +// essentially just a lexer that calls the ParseState method +// for each token. + +// The parser recognizes POSIX extended regular expressions +// excluding backreferences, collating elements, and collating +// classes. It also allows the empty string as a regular expression +// and recognizes the Perl escape sequences \d, \s, \w, \D, \S, and \W. +// See regexp.h for rationale. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "util/pod_array.h" +#include "util/strutil.h" +#include "util/utf.h" +#include "re2/regexp.h" +#include "re2/stringpiece.h" +#include "re2/unicode_casefold.h" +#include "re2/unicode_groups.h" +#include "re2/walker-inl.h" + +#if defined(RE2_USE_ICU) +#include "unicode/uniset.h" +#include "unicode/unistr.h" +#include "unicode/utypes.h" +#endif + +namespace re2 { + +// Reduce the maximum repeat count by an order of magnitude when fuzzing. +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +static const int kMaxRepeat = 100; +#else +static const int kMaxRepeat = 1000; +#endif + +// Regular expression parse state. +// The list of parsed regexps so far is maintained as a vector of +// Regexp pointers called the stack. Left parenthesis and vertical +// bar markers are also placed on the stack, as Regexps with +// non-standard opcodes. +// Scanning a left parenthesis causes the parser to push a left parenthesis +// marker on the stack. +// Scanning a vertical bar causes the parser to pop the stack until it finds a +// vertical bar or left parenthesis marker (not popping the marker), +// concatenate all the popped results, and push them back on +// the stack (DoConcatenation). +// Scanning a right parenthesis causes the parser to act as though it +// has seen a vertical bar, which then leaves the top of the stack in the +// form LeftParen regexp VerticalBar regexp VerticalBar ... regexp VerticalBar. +// The parser pops all this off the stack and creates an alternation of the +// regexps (DoAlternation). + +class Regexp::ParseState { + public: + ParseState(ParseFlags flags, const StringPiece& whole_regexp, + RegexpStatus* status); + ~ParseState(); + + ParseFlags flags() { return flags_; } + int rune_max() { return rune_max_; } + + // Parse methods. All public methods return a bool saying + // whether parsing should continue. If a method returns + // false, it has set fields in *status_, and the parser + // should return NULL. + + // Pushes the given regular expression onto the stack. + // Could check for too much memory used here. + bool PushRegexp(Regexp* re); + + // Pushes the literal rune r onto the stack. + bool PushLiteral(Rune r); + + // Pushes a regexp with the given op (and no args) onto the stack. + bool PushSimpleOp(RegexpOp op); + + // Pushes a ^ onto the stack. + bool PushCarat(); + + // Pushes a \b (word == true) or \B (word == false) onto the stack. + bool PushWordBoundary(bool word); + + // Pushes a $ onto the stack. + bool PushDollar(); + + // Pushes a . onto the stack + bool PushDot(); + + // Pushes a repeat operator regexp onto the stack. + // A valid argument for the operator must already be on the stack. + // s is the name of the operator, for use in error messages. + bool PushRepeatOp(RegexpOp op, const StringPiece& s, bool nongreedy); + + // Pushes a repetition regexp onto the stack. + // A valid argument for the operator must already be on the stack. + bool PushRepetition(int min, int max, const StringPiece& s, bool nongreedy); + + // Checks whether a particular regexp op is a marker. + bool IsMarker(RegexpOp op); + + // Processes a left parenthesis in the input. + // Pushes a marker onto the stack. + bool DoLeftParen(const StringPiece& name); + bool DoLeftParenNoCapture(); + + // Processes a vertical bar in the input. + bool DoVerticalBar(); + + // Processes a right parenthesis in the input. + bool DoRightParen(); + + // Processes the end of input, returning the final regexp. + Regexp* DoFinish(); + + // Finishes the regexp if necessary, preparing it for use + // in a more complicated expression. + // If it is a CharClassBuilder, converts into a CharClass. + Regexp* FinishRegexp(Regexp*); + + // These routines don't manipulate the parse stack + // directly, but they do need to look at flags_. + // ParseCharClass also manipulates the internals of Regexp + // while creating *out_re. + + // Parse a character class into *out_re. + // Removes parsed text from s. + bool ParseCharClass(StringPiece* s, Regexp** out_re, + RegexpStatus* status); + + // Parse a character class character into *rp. + // Removes parsed text from s. + bool ParseCCCharacter(StringPiece* s, Rune *rp, + const StringPiece& whole_class, + RegexpStatus* status); + + // Parse a character class range into rr. + // Removes parsed text from s. + bool ParseCCRange(StringPiece* s, RuneRange* rr, + const StringPiece& whole_class, + RegexpStatus* status); + + // Parse a Perl flag set or non-capturing group from s. + bool ParsePerlFlags(StringPiece* s); + + + // Finishes the current concatenation, + // collapsing it into a single regexp on the stack. + void DoConcatenation(); + + // Finishes the current alternation, + // collapsing it to a single regexp on the stack. + void DoAlternation(); + + // Generalized DoAlternation/DoConcatenation. + void DoCollapse(RegexpOp op); + + // Maybe concatenate Literals into LiteralString. + bool MaybeConcatString(int r, ParseFlags flags); + +private: + ParseFlags flags_; + StringPiece whole_regexp_; + RegexpStatus* status_; + Regexp* stacktop_; + int ncap_; // number of capturing parens seen + int rune_max_; // maximum char value for this encoding + + ParseState(const ParseState&) = delete; + ParseState& operator=(const ParseState&) = delete; +}; + +// Pseudo-operators - only on parse stack. +const RegexpOp kLeftParen = static_cast(kMaxRegexpOp+1); +const RegexpOp kVerticalBar = static_cast(kMaxRegexpOp+2); + +Regexp::ParseState::ParseState(ParseFlags flags, + const StringPiece& whole_regexp, + RegexpStatus* status) + : flags_(flags), whole_regexp_(whole_regexp), + status_(status), stacktop_(NULL), ncap_(0) { + if (flags_ & Latin1) + rune_max_ = 0xFF; + else + rune_max_ = Runemax; +} + +// Cleans up by freeing all the regexps on the stack. +Regexp::ParseState::~ParseState() { + Regexp* next; + for (Regexp* re = stacktop_; re != NULL; re = next) { + next = re->down_; + re->down_ = NULL; + if (re->op() == kLeftParen) + delete re->name_; + re->Decref(); + } +} + +// Finishes the regexp if necessary, preparing it for use in +// a more complex expression. +// If it is a CharClassBuilder, converts into a CharClass. +Regexp* Regexp::ParseState::FinishRegexp(Regexp* re) { + if (re == NULL) + return NULL; + re->down_ = NULL; + + if (re->op_ == kRegexpCharClass && re->ccb_ != NULL) { + CharClassBuilder* ccb = re->ccb_; + re->ccb_ = NULL; + re->cc_ = ccb->GetCharClass(); + delete ccb; + } + + return re; +} + +// Pushes the given regular expression onto the stack. +// Could check for too much memory used here. +bool Regexp::ParseState::PushRegexp(Regexp* re) { + MaybeConcatString(-1, NoParseFlags); + + // Special case: a character class of one character is just + // a literal. This is a common idiom for escaping + // single characters (e.g., [.] instead of \.), and some + // analysis does better with fewer character classes. + // Similarly, [Aa] can be rewritten as a literal A with ASCII case folding. + if (re->op_ == kRegexpCharClass && re->ccb_ != NULL) { + re->ccb_->RemoveAbove(rune_max_); + if (re->ccb_->size() == 1) { + Rune r = re->ccb_->begin()->lo; + re->Decref(); + re = new Regexp(kRegexpLiteral, flags_); + re->rune_ = r; + } else if (re->ccb_->size() == 2) { + Rune r = re->ccb_->begin()->lo; + if ('A' <= r && r <= 'Z' && re->ccb_->Contains(r + 'a' - 'A')) { + re->Decref(); + re = new Regexp(kRegexpLiteral, flags_ | FoldCase); + re->rune_ = r + 'a' - 'A'; + } + } + } + + if (!IsMarker(re->op())) + re->simple_ = re->ComputeSimple(); + re->down_ = stacktop_; + stacktop_ = re; + return true; +} + +// Searches the case folding tables and returns the CaseFold* that contains r. +// If there isn't one, returns the CaseFold* with smallest f->lo bigger than r. +// If there isn't one, returns NULL. +const CaseFold* LookupCaseFold(const CaseFold *f, int n, Rune r) { + const CaseFold* ef = f + n; + + // Binary search for entry containing r. + while (n > 0) { + int m = n/2; + if (f[m].lo <= r && r <= f[m].hi) + return &f[m]; + if (r < f[m].lo) { + n = m; + } else { + f += m+1; + n -= m+1; + } + } + + // There is no entry that contains r, but f points + // where it would have been. Unless f points at + // the end of the array, it points at the next entry + // after r. + if (f < ef) + return f; + + // No entry contains r; no entry contains runes > r. + return NULL; +} + +// Returns the result of applying the fold f to the rune r. +Rune ApplyFold(const CaseFold *f, Rune r) { + switch (f->delta) { + default: + return r + f->delta; + + case EvenOddSkip: // even <-> odd but only applies to every other + if ((r - f->lo) % 2) + return r; + FALLTHROUGH_INTENDED; + case EvenOdd: // even <-> odd + if (r%2 == 0) + return r + 1; + return r - 1; + + case OddEvenSkip: // odd <-> even but only applies to every other + if ((r - f->lo) % 2) + return r; + FALLTHROUGH_INTENDED; + case OddEven: // odd <-> even + if (r%2 == 1) + return r + 1; + return r - 1; + } +} + +// Returns the next Rune in r's folding cycle (see unicode_casefold.h). +// Examples: +// CycleFoldRune('A') = 'a' +// CycleFoldRune('a') = 'A' +// +// CycleFoldRune('K') = 'k' +// CycleFoldRune('k') = 0x212A (Kelvin) +// CycleFoldRune(0x212A) = 'K' +// +// CycleFoldRune('?') = '?' +Rune CycleFoldRune(Rune r) { + const CaseFold* f = LookupCaseFold(unicode_casefold, num_unicode_casefold, r); + if (f == NULL || r < f->lo) + return r; + return ApplyFold(f, r); +} + +// Add lo-hi to the class, along with their fold-equivalent characters. +// If lo-hi is already in the class, assume that the fold-equivalent +// chars are there too, so there's no work to do. +static void AddFoldedRange(CharClassBuilder* cc, Rune lo, Rune hi, int depth) { + // AddFoldedRange calls itself recursively for each rune in the fold cycle. + // Most folding cycles are small: there aren't any bigger than four in the + // current Unicode tables. make_unicode_casefold.py checks that + // the cycles are not too long, and we double-check here using depth. + if (depth > 10) { + LOG(DFATAL) << "AddFoldedRange recurses too much."; + return; + } + + if (!cc->AddRange(lo, hi)) // lo-hi was already there? we're done + return; + + while (lo <= hi) { + const CaseFold* f = LookupCaseFold(unicode_casefold, num_unicode_casefold, lo); + if (f == NULL) // lo has no fold, nor does anything above lo + break; + if (lo < f->lo) { // lo has no fold; next rune with a fold is f->lo + lo = f->lo; + continue; + } + + // Add in the result of folding the range lo - f->hi + // and that range's fold, recursively. + Rune lo1 = lo; + Rune hi1 = std::min(hi, f->hi); + switch (f->delta) { + default: + lo1 += f->delta; + hi1 += f->delta; + break; + case EvenOdd: + if (lo1%2 == 1) + lo1--; + if (hi1%2 == 0) + hi1++; + break; + case OddEven: + if (lo1%2 == 0) + lo1--; + if (hi1%2 == 1) + hi1++; + break; + } + AddFoldedRange(cc, lo1, hi1, depth+1); + + // Pick up where this fold left off. + lo = f->hi + 1; + } +} + +// Pushes the literal rune r onto the stack. +bool Regexp::ParseState::PushLiteral(Rune r) { + // Do case folding if needed. + if ((flags_ & FoldCase) && CycleFoldRune(r) != r) { + Regexp* re = new Regexp(kRegexpCharClass, flags_ & ~FoldCase); + re->ccb_ = new CharClassBuilder; + Rune r1 = r; + do { + if (!(flags_ & NeverNL) || r != '\n') { + re->ccb_->AddRange(r, r); + } + r = CycleFoldRune(r); + } while (r != r1); + return PushRegexp(re); + } + + // Exclude newline if applicable. + if ((flags_ & NeverNL) && r == '\n') + return PushRegexp(new Regexp(kRegexpNoMatch, flags_)); + + // No fancy stuff worked. Ordinary literal. + if (MaybeConcatString(r, flags_)) + return true; + + Regexp* re = new Regexp(kRegexpLiteral, flags_); + re->rune_ = r; + return PushRegexp(re); +} + +// Pushes a ^ onto the stack. +bool Regexp::ParseState::PushCarat() { + if (flags_ & OneLine) { + return PushSimpleOp(kRegexpBeginText); + } + return PushSimpleOp(kRegexpBeginLine); +} + +// Pushes a \b or \B onto the stack. +bool Regexp::ParseState::PushWordBoundary(bool word) { + if (word) + return PushSimpleOp(kRegexpWordBoundary); + return PushSimpleOp(kRegexpNoWordBoundary); +} + +// Pushes a $ onto the stack. +bool Regexp::ParseState::PushDollar() { + if (flags_ & OneLine) { + // Clumsy marker so that MimicsPCRE() can tell whether + // this kRegexpEndText was a $ and not a \z. + Regexp::ParseFlags oflags = flags_; + flags_ = flags_ | WasDollar; + bool ret = PushSimpleOp(kRegexpEndText); + flags_ = oflags; + return ret; + } + return PushSimpleOp(kRegexpEndLine); +} + +// Pushes a . onto the stack. +bool Regexp::ParseState::PushDot() { + if ((flags_ & DotNL) && !(flags_ & NeverNL)) + return PushSimpleOp(kRegexpAnyChar); + // Rewrite . into [^\n] + Regexp* re = new Regexp(kRegexpCharClass, flags_ & ~FoldCase); + re->ccb_ = new CharClassBuilder; + re->ccb_->AddRange(0, '\n' - 1); + re->ccb_->AddRange('\n' + 1, rune_max_); + return PushRegexp(re); +} + +// Pushes a regexp with the given op (and no args) onto the stack. +bool Regexp::ParseState::PushSimpleOp(RegexpOp op) { + Regexp* re = new Regexp(op, flags_); + return PushRegexp(re); +} + +// Pushes a repeat operator regexp onto the stack. +// A valid argument for the operator must already be on the stack. +// The char c is the name of the operator, for use in error messages. +bool Regexp::ParseState::PushRepeatOp(RegexpOp op, const StringPiece& s, + bool nongreedy) { + if (stacktop_ == NULL || IsMarker(stacktop_->op())) { + status_->set_code(kRegexpRepeatArgument); + status_->set_error_arg(s); + return false; + } + Regexp::ParseFlags fl = flags_; + if (nongreedy) + fl = fl ^ NonGreedy; + + // Squash **, ++ and ??. Regexp::Star() et al. handle this too, but + // they're mostly for use during simplification, not during parsing. + if (op == stacktop_->op() && fl == stacktop_->parse_flags()) + return true; + + // Squash *+, *?, +*, +?, ?* and ?+. They all squash to *, so because + // op is a repeat, we just have to check that stacktop_->op() is too, + // then adjust stacktop_. + if ((stacktop_->op() == kRegexpStar || + stacktop_->op() == kRegexpPlus || + stacktop_->op() == kRegexpQuest) && + fl == stacktop_->parse_flags()) { + stacktop_->op_ = kRegexpStar; + return true; + } + + Regexp* re = new Regexp(op, fl); + re->AllocSub(1); + re->down_ = stacktop_->down_; + re->sub()[0] = FinishRegexp(stacktop_); + re->simple_ = re->ComputeSimple(); + stacktop_ = re; + return true; +} + +// RepetitionWalker reports whether the repetition regexp is valid. +// Valid means that the combination of the top-level repetition +// and any inner repetitions does not exceed n copies of the +// innermost thing. +// This rewalks the regexp tree and is called for every repetition, +// so we have to worry about inducing quadratic behavior in the parser. +// We avoid this by only using RepetitionWalker when min or max >= 2. +// In that case the depth of any >= 2 nesting can only get to 9 without +// triggering a parse error, so each subtree can only be rewalked 9 times. +class RepetitionWalker : public Regexp::Walker { + public: + RepetitionWalker() {} + virtual int PreVisit(Regexp* re, int parent_arg, bool* stop); + virtual int PostVisit(Regexp* re, int parent_arg, int pre_arg, + int* child_args, int nchild_args); + virtual int ShortVisit(Regexp* re, int parent_arg); + + private: + RepetitionWalker(const RepetitionWalker&) = delete; + RepetitionWalker& operator=(const RepetitionWalker&) = delete; +}; + +int RepetitionWalker::PreVisit(Regexp* re, int parent_arg, bool* stop) { + int arg = parent_arg; + if (re->op() == kRegexpRepeat) { + int m = re->max(); + if (m < 0) { + m = re->min(); + } + if (m > 0) { + arg /= m; + } + } + return arg; +} + +int RepetitionWalker::PostVisit(Regexp* re, int parent_arg, int pre_arg, + int* child_args, int nchild_args) { + int arg = pre_arg; + for (int i = 0; i < nchild_args; i++) { + if (child_args[i] < arg) { + arg = child_args[i]; + } + } + return arg; +} + +int RepetitionWalker::ShortVisit(Regexp* re, int parent_arg) { + // This should never be called, since we use Walk and not + // WalkExponential. + LOG(DFATAL) << "RepetitionWalker::ShortVisit called"; + return 0; +} + +// Pushes a repetition regexp onto the stack. +// A valid argument for the operator must already be on the stack. +bool Regexp::ParseState::PushRepetition(int min, int max, + const StringPiece& s, + bool nongreedy) { + if ((max != -1 && max < min) || min > kMaxRepeat || max > kMaxRepeat) { + status_->set_code(kRegexpRepeatSize); + status_->set_error_arg(s); + return false; + } + if (stacktop_ == NULL || IsMarker(stacktop_->op())) { + status_->set_code(kRegexpRepeatArgument); + status_->set_error_arg(s); + return false; + } + Regexp::ParseFlags fl = flags_; + if (nongreedy) + fl = fl ^ NonGreedy; + Regexp* re = new Regexp(kRegexpRepeat, fl); + re->min_ = min; + re->max_ = max; + re->AllocSub(1); + re->down_ = stacktop_->down_; + re->sub()[0] = FinishRegexp(stacktop_); + re->simple_ = re->ComputeSimple(); + stacktop_ = re; + if (min >= 2 || max >= 2) { + RepetitionWalker w; + if (w.Walk(stacktop_, kMaxRepeat) == 0) { + status_->set_code(kRegexpRepeatSize); + status_->set_error_arg(s); + return false; + } + } + return true; +} + +// Checks whether a particular regexp op is a marker. +bool Regexp::ParseState::IsMarker(RegexpOp op) { + return op >= kLeftParen; +} + +// Processes a left parenthesis in the input. +// Pushes a marker onto the stack. +bool Regexp::ParseState::DoLeftParen(const StringPiece& name) { + Regexp* re = new Regexp(kLeftParen, flags_); + re->cap_ = ++ncap_; + if (name.data() != NULL) + re->name_ = new std::string(name); + return PushRegexp(re); +} + +// Pushes a non-capturing marker onto the stack. +bool Regexp::ParseState::DoLeftParenNoCapture() { + Regexp* re = new Regexp(kLeftParen, flags_); + re->cap_ = -1; + return PushRegexp(re); +} + +// Processes a vertical bar in the input. +bool Regexp::ParseState::DoVerticalBar() { + MaybeConcatString(-1, NoParseFlags); + DoConcatenation(); + + // Below the vertical bar is a list to alternate. + // Above the vertical bar is a list to concatenate. + // We just did the concatenation, so either swap + // the result below the vertical bar or push a new + // vertical bar on the stack. + Regexp* r1; + Regexp* r2; + if ((r1 = stacktop_) != NULL && + (r2 = r1->down_) != NULL && + r2->op() == kVerticalBar) { + Regexp* r3; + if ((r3 = r2->down_) != NULL && + (r1->op() == kRegexpAnyChar || r3->op() == kRegexpAnyChar)) { + // AnyChar is above or below the vertical bar. Let it subsume + // the other when the other is Literal, CharClass or AnyChar. + if (r3->op() == kRegexpAnyChar && + (r1->op() == kRegexpLiteral || + r1->op() == kRegexpCharClass || + r1->op() == kRegexpAnyChar)) { + // Discard r1. + stacktop_ = r2; + r1->Decref(); + return true; + } + if (r1->op() == kRegexpAnyChar && + (r3->op() == kRegexpLiteral || + r3->op() == kRegexpCharClass || + r3->op() == kRegexpAnyChar)) { + // Rearrange the stack and discard r3. + r1->down_ = r3->down_; + r2->down_ = r1; + stacktop_ = r2; + r3->Decref(); + return true; + } + } + // Swap r1 below vertical bar (r2). + r1->down_ = r2->down_; + r2->down_ = r1; + stacktop_ = r2; + return true; + } + return PushSimpleOp(kVerticalBar); +} + +// Processes a right parenthesis in the input. +bool Regexp::ParseState::DoRightParen() { + // Finish the current concatenation and alternation. + DoAlternation(); + + // The stack should be: LeftParen regexp + // Remove the LeftParen, leaving the regexp, + // parenthesized. + Regexp* r1; + Regexp* r2; + if ((r1 = stacktop_) == NULL || + (r2 = r1->down_) == NULL || + r2->op() != kLeftParen) { + status_->set_code(kRegexpMissingParen); + status_->set_error_arg(whole_regexp_); + return false; + } + + // Pop off r1, r2. Will Decref or reuse below. + stacktop_ = r2->down_; + + // Restore flags from when paren opened. + Regexp* re = r2; + flags_ = re->parse_flags(); + + // Rewrite LeftParen as capture if needed. + if (re->cap_ > 0) { + re->op_ = kRegexpCapture; + // re->cap_ is already set + re->AllocSub(1); + re->sub()[0] = FinishRegexp(r1); + re->simple_ = re->ComputeSimple(); + } else { + re->Decref(); + re = r1; + } + return PushRegexp(re); +} + +// Processes the end of input, returning the final regexp. +Regexp* Regexp::ParseState::DoFinish() { + DoAlternation(); + Regexp* re = stacktop_; + if (re != NULL && re->down_ != NULL) { + status_->set_code(kRegexpMissingParen); + status_->set_error_arg(whole_regexp_); + return NULL; + } + stacktop_ = NULL; + return FinishRegexp(re); +} + +// Returns the leading regexp that re starts with. +// The returned Regexp* points into a piece of re, +// so it must not be used after the caller calls re->Decref(). +Regexp* Regexp::LeadingRegexp(Regexp* re) { + if (re->op() == kRegexpEmptyMatch) + return NULL; + if (re->op() == kRegexpConcat && re->nsub() >= 2) { + Regexp** sub = re->sub(); + if (sub[0]->op() == kRegexpEmptyMatch) + return NULL; + return sub[0]; + } + return re; +} + +// Removes LeadingRegexp(re) from re and returns what's left. +// Consumes the reference to re and may edit it in place. +// If caller wants to hold on to LeadingRegexp(re), +// must have already Incref'ed it. +Regexp* Regexp::RemoveLeadingRegexp(Regexp* re) { + if (re->op() == kRegexpEmptyMatch) + return re; + if (re->op() == kRegexpConcat && re->nsub() >= 2) { + Regexp** sub = re->sub(); + if (sub[0]->op() == kRegexpEmptyMatch) + return re; + sub[0]->Decref(); + sub[0] = NULL; + if (re->nsub() == 2) { + // Collapse concatenation to single regexp. + Regexp* nre = sub[1]; + sub[1] = NULL; + re->Decref(); + return nre; + } + // 3 or more -> 2 or more. + re->nsub_--; + memmove(sub, sub + 1, re->nsub_ * sizeof sub[0]); + return re; + } + Regexp::ParseFlags pf = re->parse_flags(); + re->Decref(); + return new Regexp(kRegexpEmptyMatch, pf); +} + +// Returns the leading string that re starts with. +// The returned Rune* points into a piece of re, +// so it must not be used after the caller calls re->Decref(). +Rune* Regexp::LeadingString(Regexp* re, int *nrune, + Regexp::ParseFlags *flags) { + while (re->op() == kRegexpConcat && re->nsub() > 0) + re = re->sub()[0]; + + *flags = static_cast(re->parse_flags_ & Regexp::FoldCase); + + if (re->op() == kRegexpLiteral) { + *nrune = 1; + return &re->rune_; + } + + if (re->op() == kRegexpLiteralString) { + *nrune = re->nrunes_; + return re->runes_; + } + + *nrune = 0; + return NULL; +} + +// Removes the first n leading runes from the beginning of re. +// Edits re in place. +void Regexp::RemoveLeadingString(Regexp* re, int n) { + // Chase down concats to find first string. + // For regexps generated by parser, nested concats are + // flattened except when doing so would overflow the 16-bit + // limit on the size of a concatenation, so we should never + // see more than two here. + Regexp* stk[4]; + size_t d = 0; + while (re->op() == kRegexpConcat) { + if (d < arraysize(stk)) + stk[d++] = re; + re = re->sub()[0]; + } + + // Remove leading string from re. + if (re->op() == kRegexpLiteral) { + re->rune_ = 0; + re->op_ = kRegexpEmptyMatch; + } else if (re->op() == kRegexpLiteralString) { + if (n >= re->nrunes_) { + delete[] re->runes_; + re->runes_ = NULL; + re->nrunes_ = 0; + re->op_ = kRegexpEmptyMatch; + } else if (n == re->nrunes_ - 1) { + Rune rune = re->runes_[re->nrunes_ - 1]; + delete[] re->runes_; + re->runes_ = NULL; + re->nrunes_ = 0; + re->rune_ = rune; + re->op_ = kRegexpLiteral; + } else { + re->nrunes_ -= n; + memmove(re->runes_, re->runes_ + n, re->nrunes_ * sizeof re->runes_[0]); + } + } + + // If re is now empty, concatenations might simplify too. + while (d > 0) { + re = stk[--d]; + Regexp** sub = re->sub(); + if (sub[0]->op() == kRegexpEmptyMatch) { + sub[0]->Decref(); + sub[0] = NULL; + // Delete first element of concat. + switch (re->nsub()) { + case 0: + case 1: + // Impossible. + LOG(DFATAL) << "Concat of " << re->nsub(); + re->submany_ = NULL; + re->op_ = kRegexpEmptyMatch; + break; + + case 2: { + // Replace re with sub[1]. + Regexp* old = sub[1]; + sub[1] = NULL; + re->Swap(old); + old->Decref(); + break; + } + + default: + // Slide down. + re->nsub_--; + memmove(sub, sub + 1, re->nsub_ * sizeof sub[0]); + break; + } + } + } +} + +// In the context of factoring alternations, a Splice is: a factored prefix or +// merged character class computed by one iteration of one round of factoring; +// the span of subexpressions of the alternation to be "spliced" (i.e. removed +// and replaced); and, for a factored prefix, the number of suffixes after any +// factoring that might have subsequently been performed on them. For a merged +// character class, there are no suffixes, of course, so the field is ignored. +struct Splice { + Splice(Regexp* prefix, Regexp** sub, int nsub) + : prefix(prefix), + sub(sub), + nsub(nsub), + nsuffix(-1) {} + + Regexp* prefix; + Regexp** sub; + int nsub; + int nsuffix; +}; + +// Named so because it is used to implement an explicit stack, a Frame is: the +// span of subexpressions of the alternation to be factored; the current round +// of factoring; any Splices computed; and, for a factored prefix, an iterator +// to the next Splice to be factored (i.e. in another Frame) because suffixes. +struct Frame { + Frame(Regexp** sub, int nsub) + : sub(sub), + nsub(nsub), + round(0) {} + + Regexp** sub; + int nsub; + int round; + std::vector splices; + int spliceidx; +}; + +// Bundled into a class for friend access to Regexp without needing to declare +// (or define) Splice in regexp.h. +class FactorAlternationImpl { + public: + static void Round1(Regexp** sub, int nsub, + Regexp::ParseFlags flags, + std::vector* splices); + static void Round2(Regexp** sub, int nsub, + Regexp::ParseFlags flags, + std::vector* splices); + static void Round3(Regexp** sub, int nsub, + Regexp::ParseFlags flags, + std::vector* splices); +}; + +// Factors common prefixes from alternation. +// For example, +// ABC|ABD|AEF|BCX|BCY +// simplifies to +// A(B(C|D)|EF)|BC(X|Y) +// and thence to +// A(B[CD]|EF)|BC[XY] +// +// Rewrites sub to contain simplified list to alternate and returns +// the new length of sub. Adjusts reference counts accordingly +// (incoming sub[i] decremented, outgoing sub[i] incremented). +int Regexp::FactorAlternation(Regexp** sub, int nsub, ParseFlags flags) { + std::vector stk; + stk.emplace_back(sub, nsub); + + for (;;) { + auto& sub = stk.back().sub; + auto& nsub = stk.back().nsub; + auto& round = stk.back().round; + auto& splices = stk.back().splices; + auto& spliceidx = stk.back().spliceidx; + + if (splices.empty()) { + // Advance to the next round of factoring. Note that this covers + // the initialised state: when splices is empty and round is 0. + round++; + } else if (spliceidx < static_cast(splices.size())) { + // We have at least one more Splice to factor. Recurse logically. + stk.emplace_back(splices[spliceidx].sub, splices[spliceidx].nsub); + continue; + } else { + // We have no more Splices to factor. Apply them. + auto iter = splices.begin(); + int out = 0; + for (int i = 0; i < nsub; ) { + // Copy until we reach where the next Splice begins. + while (sub + i < iter->sub) + sub[out++] = sub[i++]; + switch (round) { + case 1: + case 2: { + // Assemble the Splice prefix and the suffixes. + Regexp* re[2]; + re[0] = iter->prefix; + re[1] = Regexp::AlternateNoFactor(iter->sub, iter->nsuffix, flags); + sub[out++] = Regexp::Concat(re, 2, flags); + i += iter->nsub; + break; + } + case 3: + // Just use the Splice prefix. + sub[out++] = iter->prefix; + i += iter->nsub; + break; + default: + LOG(DFATAL) << "unknown round: " << round; + break; + } + // If we are done, copy until the end of sub. + if (++iter == splices.end()) { + while (i < nsub) + sub[out++] = sub[i++]; + } + } + splices.clear(); + nsub = out; + // Advance to the next round of factoring. + round++; + } + + switch (round) { + case 1: + FactorAlternationImpl::Round1(sub, nsub, flags, &splices); + break; + case 2: + FactorAlternationImpl::Round2(sub, nsub, flags, &splices); + break; + case 3: + FactorAlternationImpl::Round3(sub, nsub, flags, &splices); + break; + case 4: + if (stk.size() == 1) { + // We are at the top of the stack. Just return. + return nsub; + } else { + // Pop the stack and set the number of suffixes. + // (Note that references will be invalidated!) + int nsuffix = nsub; + stk.pop_back(); + stk.back().splices[stk.back().spliceidx].nsuffix = nsuffix; + ++stk.back().spliceidx; + continue; + } + default: + LOG(DFATAL) << "unknown round: " << round; + break; + } + + // Set spliceidx depending on whether we have Splices to factor. + if (splices.empty() || round == 3) { + spliceidx = static_cast(splices.size()); + } else { + spliceidx = 0; + } + } +} + +void FactorAlternationImpl::Round1(Regexp** sub, int nsub, + Regexp::ParseFlags flags, + std::vector* splices) { + // Round 1: Factor out common literal prefixes. + int start = 0; + Rune* rune = NULL; + int nrune = 0; + Regexp::ParseFlags runeflags = Regexp::NoParseFlags; + for (int i = 0; i <= nsub; i++) { + // Invariant: sub[start:i] consists of regexps that all + // begin with rune[0:nrune]. + Rune* rune_i = NULL; + int nrune_i = 0; + Regexp::ParseFlags runeflags_i = Regexp::NoParseFlags; + if (i < nsub) { + rune_i = Regexp::LeadingString(sub[i], &nrune_i, &runeflags_i); + if (runeflags_i == runeflags) { + int same = 0; + while (same < nrune && same < nrune_i && rune[same] == rune_i[same]) + same++; + if (same > 0) { + // Matches at least one rune in current range. Keep going around. + nrune = same; + continue; + } + } + } + + // Found end of a run with common leading literal string: + // sub[start:i] all begin with rune[0:nrune], + // but sub[i] does not even begin with rune[0]. + if (i == start) { + // Nothing to do - first iteration. + } else if (i == start+1) { + // Just one: don't bother factoring. + } else { + Regexp* prefix = Regexp::LiteralString(rune, nrune, runeflags); + for (int j = start; j < i; j++) + Regexp::RemoveLeadingString(sub[j], nrune); + splices->emplace_back(prefix, sub + start, i - start); + } + + // Prepare for next iteration (if there is one). + if (i < nsub) { + start = i; + rune = rune_i; + nrune = nrune_i; + runeflags = runeflags_i; + } + } +} + +void FactorAlternationImpl::Round2(Regexp** sub, int nsub, + Regexp::ParseFlags flags, + std::vector* splices) { + // Round 2: Factor out common simple prefixes, + // just the first piece of each concatenation. + // This will be good enough a lot of the time. + // + // Complex subexpressions (e.g. involving quantifiers) + // are not safe to factor because that collapses their + // distinct paths through the automaton, which affects + // correctness in some cases. + int start = 0; + Regexp* first = NULL; + for (int i = 0; i <= nsub; i++) { + // Invariant: sub[start:i] consists of regexps that all + // begin with first. + Regexp* first_i = NULL; + if (i < nsub) { + first_i = Regexp::LeadingRegexp(sub[i]); + if (first != NULL && + // first must be an empty-width op + // OR a char class, any char or any byte + // OR a fixed repeat of a literal, char class, any char or any byte. + (first->op() == kRegexpBeginLine || + first->op() == kRegexpEndLine || + first->op() == kRegexpWordBoundary || + first->op() == kRegexpNoWordBoundary || + first->op() == kRegexpBeginText || + first->op() == kRegexpEndText || + first->op() == kRegexpCharClass || + first->op() == kRegexpAnyChar || + first->op() == kRegexpAnyByte || + (first->op() == kRegexpRepeat && + first->min() == first->max() && + (first->sub()[0]->op() == kRegexpLiteral || + first->sub()[0]->op() == kRegexpCharClass || + first->sub()[0]->op() == kRegexpAnyChar || + first->sub()[0]->op() == kRegexpAnyByte))) && + Regexp::Equal(first, first_i)) + continue; + } + + // Found end of a run with common leading regexp: + // sub[start:i] all begin with first, + // but sub[i] does not. + if (i == start) { + // Nothing to do - first iteration. + } else if (i == start+1) { + // Just one: don't bother factoring. + } else { + Regexp* prefix = first->Incref(); + for (int j = start; j < i; j++) + sub[j] = Regexp::RemoveLeadingRegexp(sub[j]); + splices->emplace_back(prefix, sub + start, i - start); + } + + // Prepare for next iteration (if there is one). + if (i < nsub) { + start = i; + first = first_i; + } + } +} + +void FactorAlternationImpl::Round3(Regexp** sub, int nsub, + Regexp::ParseFlags flags, + std::vector* splices) { + // Round 3: Merge runs of literals and/or character classes. + int start = 0; + Regexp* first = NULL; + for (int i = 0; i <= nsub; i++) { + // Invariant: sub[start:i] consists of regexps that all + // are either literals (i.e. runes) or character classes. + Regexp* first_i = NULL; + if (i < nsub) { + first_i = sub[i]; + if (first != NULL && + (first->op() == kRegexpLiteral || + first->op() == kRegexpCharClass) && + (first_i->op() == kRegexpLiteral || + first_i->op() == kRegexpCharClass)) + continue; + } + + // Found end of a run of Literal/CharClass: + // sub[start:i] all are either one or the other, + // but sub[i] is not. + if (i == start) { + // Nothing to do - first iteration. + } else if (i == start+1) { + // Just one: don't bother factoring. + } else { + CharClassBuilder ccb; + for (int j = start; j < i; j++) { + Regexp* re = sub[j]; + if (re->op() == kRegexpCharClass) { + CharClass* cc = re->cc(); + for (CharClass::iterator it = cc->begin(); it != cc->end(); ++it) + ccb.AddRange(it->lo, it->hi); + } else if (re->op() == kRegexpLiteral) { + ccb.AddRangeFlags(re->rune(), re->rune(), re->parse_flags()); + } else { + LOG(DFATAL) << "RE2: unexpected op: " << re->op() << " " + << re->ToString(); + } + re->Decref(); + } + Regexp* re = Regexp::NewCharClass(ccb.GetCharClass(), flags); + splices->emplace_back(re, sub + start, i - start); + } + + // Prepare for next iteration (if there is one). + if (i < nsub) { + start = i; + first = first_i; + } + } +} + +// Collapse the regexps on top of the stack, down to the +// first marker, into a new op node (op == kRegexpAlternate +// or op == kRegexpConcat). +void Regexp::ParseState::DoCollapse(RegexpOp op) { + // Scan backward to marker, counting children of composite. + int n = 0; + Regexp* next = NULL; + Regexp* sub; + for (sub = stacktop_; sub != NULL && !IsMarker(sub->op()); sub = next) { + next = sub->down_; + if (sub->op_ == op) + n += sub->nsub_; + else + n++; + } + + // If there's just one child, leave it alone. + // (Concat of one thing is that one thing; alternate of one thing is same.) + if (stacktop_ != NULL && stacktop_->down_ == next) + return; + + // Construct op (alternation or concatenation), flattening op of op. + PODArray subs(n); + next = NULL; + int i = n; + for (sub = stacktop_; sub != NULL && !IsMarker(sub->op()); sub = next) { + next = sub->down_; + if (sub->op_ == op) { + Regexp** sub_subs = sub->sub(); + for (int k = sub->nsub_ - 1; k >= 0; k--) + subs[--i] = sub_subs[k]->Incref(); + sub->Decref(); + } else { + subs[--i] = FinishRegexp(sub); + } + } + + Regexp* re = ConcatOrAlternate(op, subs.data(), n, flags_, true); + re->simple_ = re->ComputeSimple(); + re->down_ = next; + stacktop_ = re; +} + +// Finishes the current concatenation, +// collapsing it into a single regexp on the stack. +void Regexp::ParseState::DoConcatenation() { + Regexp* r1 = stacktop_; + if (r1 == NULL || IsMarker(r1->op())) { + // empty concatenation is special case + Regexp* re = new Regexp(kRegexpEmptyMatch, flags_); + PushRegexp(re); + } + DoCollapse(kRegexpConcat); +} + +// Finishes the current alternation, +// collapsing it to a single regexp on the stack. +void Regexp::ParseState::DoAlternation() { + DoVerticalBar(); + // Now stack top is kVerticalBar. + Regexp* r1 = stacktop_; + stacktop_ = r1->down_; + r1->Decref(); + DoCollapse(kRegexpAlternate); +} + +// Incremental conversion of concatenated literals into strings. +// If top two elements on stack are both literal or string, +// collapse into single string. +// Don't walk down the stack -- the parser calls this frequently +// enough that below the bottom two is known to be collapsed. +// Only called when another regexp is about to be pushed +// on the stack, so that the topmost literal is not being considered. +// (Otherwise ab* would turn into (ab)*.) +// If r >= 0, consider pushing a literal r on the stack. +// Return whether that happened. +bool Regexp::ParseState::MaybeConcatString(int r, ParseFlags flags) { + Regexp* re1; + Regexp* re2; + if ((re1 = stacktop_) == NULL || (re2 = re1->down_) == NULL) + return false; + + if (re1->op_ != kRegexpLiteral && re1->op_ != kRegexpLiteralString) + return false; + if (re2->op_ != kRegexpLiteral && re2->op_ != kRegexpLiteralString) + return false; + if ((re1->parse_flags_ & FoldCase) != (re2->parse_flags_ & FoldCase)) + return false; + + if (re2->op_ == kRegexpLiteral) { + // convert into string + Rune rune = re2->rune_; + re2->op_ = kRegexpLiteralString; + re2->nrunes_ = 0; + re2->runes_ = NULL; + re2->AddRuneToString(rune); + } + + // push re1 into re2. + if (re1->op_ == kRegexpLiteral) { + re2->AddRuneToString(re1->rune_); + } else { + for (int i = 0; i < re1->nrunes_; i++) + re2->AddRuneToString(re1->runes_[i]); + re1->nrunes_ = 0; + delete[] re1->runes_; + re1->runes_ = NULL; + } + + // reuse re1 if possible + if (r >= 0) { + re1->op_ = kRegexpLiteral; + re1->rune_ = r; + re1->parse_flags_ = static_cast(flags); + return true; + } + + stacktop_ = re2; + re1->Decref(); + return false; +} + +// Lexing routines. + +// Parses a decimal integer, storing it in *np. +// Sets *s to span the remainder of the string. +static bool ParseInteger(StringPiece* s, int* np) { + if (s->size() == 0 || !isdigit((*s)[0] & 0xFF)) + return false; + // Disallow leading zeros. + if (s->size() >= 2 && (*s)[0] == '0' && isdigit((*s)[1] & 0xFF)) + return false; + int n = 0; + int c; + while (s->size() > 0 && isdigit(c = (*s)[0] & 0xFF)) { + // Avoid overflow. + if (n >= 100000000) + return false; + n = n*10 + c - '0'; + s->remove_prefix(1); // digit + } + *np = n; + return true; +} + +// Parses a repetition suffix like {1,2} or {2} or {2,}. +// Sets *s to span the remainder of the string on success. +// Sets *lo and *hi to the given range. +// In the case of {2,}, the high number is unbounded; +// sets *hi to -1 to signify this. +// {,2} is NOT a valid suffix. +// The Maybe in the name signifies that the regexp parse +// doesn't fail even if ParseRepetition does, so the StringPiece +// s must NOT be edited unless MaybeParseRepetition returns true. +static bool MaybeParseRepetition(StringPiece* sp, int* lo, int* hi) { + StringPiece s = *sp; + if (s.size() == 0 || s[0] != '{') + return false; + s.remove_prefix(1); // '{' + if (!ParseInteger(&s, lo)) + return false; + if (s.size() == 0) + return false; + if (s[0] == ',') { + s.remove_prefix(1); // ',' + if (s.size() == 0) + return false; + if (s[0] == '}') { + // {2,} means at least 2 + *hi = -1; + } else { + // {2,4} means 2, 3, or 4. + if (!ParseInteger(&s, hi)) + return false; + } + } else { + // {2} means exactly two + *hi = *lo; + } + if (s.size() == 0 || s[0] != '}') + return false; + s.remove_prefix(1); // '}' + *sp = s; + return true; +} + +// Removes the next Rune from the StringPiece and stores it in *r. +// Returns number of bytes removed from sp. +// Behaves as though there is a terminating NUL at the end of sp. +// Argument order is backwards from usual Google style +// but consistent with chartorune. +static int StringPieceToRune(Rune *r, StringPiece *sp, RegexpStatus* status) { + // fullrune() takes int, not size_t. However, it just looks + // at the leading byte and treats any length >= 4 the same. + if (fullrune(sp->data(), static_cast(std::min(size_t{4}, sp->size())))) { + int n = chartorune(r, sp->data()); + // Some copies of chartorune have a bug that accepts + // encodings of values in (10FFFF, 1FFFFF] as valid. + // Those values break the character class algorithm, + // which assumes Runemax is the largest rune. + if (*r > Runemax) { + n = 1; + *r = Runeerror; + } + if (!(n == 1 && *r == Runeerror)) { // no decoding error + sp->remove_prefix(n); + return n; + } + } + + status->set_code(kRegexpBadUTF8); + status->set_error_arg(StringPiece()); + return -1; +} + +// Return whether name is valid UTF-8. +// If not, set status to kRegexpBadUTF8. +static bool IsValidUTF8(const StringPiece& s, RegexpStatus* status) { + StringPiece t = s; + Rune r; + while (t.size() > 0) { + if (StringPieceToRune(&r, &t, status) < 0) + return false; + } + return true; +} + +// Is c a hex digit? +static int IsHex(int c) { + return ('0' <= c && c <= '9') || + ('A' <= c && c <= 'F') || + ('a' <= c && c <= 'f'); +} + +// Convert hex digit to value. +static int UnHex(int c) { + if ('0' <= c && c <= '9') + return c - '0'; + if ('A' <= c && c <= 'F') + return c - 'A' + 10; + if ('a' <= c && c <= 'f') + return c - 'a' + 10; + LOG(DFATAL) << "Bad hex digit " << c; + return 0; +} + +// Parse an escape sequence (e.g., \n, \{). +// Sets *s to span the remainder of the string. +// Sets *rp to the named character. +static bool ParseEscape(StringPiece* s, Rune* rp, + RegexpStatus* status, int rune_max) { + const char* begin = s->begin(); + if (s->size() < 1 || (*s)[0] != '\\') { + // Should not happen - caller always checks. + status->set_code(kRegexpInternalError); + status->set_error_arg(StringPiece()); + return false; + } + if (s->size() < 2) { + status->set_code(kRegexpTrailingBackslash); + status->set_error_arg(StringPiece()); + return false; + } + Rune c, c1; + s->remove_prefix(1); // backslash + if (StringPieceToRune(&c, s, status) < 0) + return false; + int code; + switch (c) { + default: + if (c < Runeself && !isalpha(c) && !isdigit(c)) { + // Escaped non-word characters are always themselves. + // PCRE is not quite so rigorous: it accepts things like + // \q, but we don't. We once rejected \_, but too many + // programs and people insist on using it, so allow \_. + *rp = c; + return true; + } + goto BadEscape; + + // Octal escapes. + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + // Single non-zero octal digit is a backreference; not supported. + if (s->size() == 0 || (*s)[0] < '0' || (*s)[0] > '7') + goto BadEscape; + FALLTHROUGH_INTENDED; + case '0': + // consume up to three octal digits; already have one. + code = c - '0'; + if (s->size() > 0 && '0' <= (c = (*s)[0]) && c <= '7') { + code = code * 8 + c - '0'; + s->remove_prefix(1); // digit + if (s->size() > 0) { + c = (*s)[0]; + if ('0' <= c && c <= '7') { + code = code * 8 + c - '0'; + s->remove_prefix(1); // digit + } + } + } + if (code > rune_max) + goto BadEscape; + *rp = code; + return true; + + // Hexadecimal escapes + case 'x': + if (s->size() == 0) + goto BadEscape; + if (StringPieceToRune(&c, s, status) < 0) + return false; + if (c == '{') { + // Any number of digits in braces. + // Update n as we consume the string, so that + // the whole thing gets shown in the error message. + // Perl accepts any text at all; it ignores all text + // after the first non-hex digit. We require only hex digits, + // and at least one. + if (StringPieceToRune(&c, s, status) < 0) + return false; + int nhex = 0; + code = 0; + while (IsHex(c)) { + nhex++; + code = code * 16 + UnHex(c); + if (code > rune_max) + goto BadEscape; + if (s->size() == 0) + goto BadEscape; + if (StringPieceToRune(&c, s, status) < 0) + return false; + } + if (c != '}' || nhex == 0) + goto BadEscape; + *rp = code; + return true; + } + // Easy case: two hex digits. + if (s->size() == 0) + goto BadEscape; + if (StringPieceToRune(&c1, s, status) < 0) + return false; + if (!IsHex(c) || !IsHex(c1)) + goto BadEscape; + *rp = UnHex(c) * 16 + UnHex(c1); + return true; + + // C escapes. + case 'n': + *rp = '\n'; + return true; + case 'r': + *rp = '\r'; + return true; + case 't': + *rp = '\t'; + return true; + + // Less common C escapes. + case 'a': + *rp = '\a'; + return true; + case 'f': + *rp = '\f'; + return true; + case 'v': + *rp = '\v'; + return true; + + // This code is disabled to avoid misparsing + // the Perl word-boundary \b as a backspace + // when in POSIX regexp mode. Surprisingly, + // in Perl, \b means word-boundary but [\b] + // means backspace. We don't support that: + // if you want a backspace embed a literal + // backspace character or use \x08. + // + // case 'b': + // *rp = '\b'; + // return true; + } + + LOG(DFATAL) << "Not reached in ParseEscape."; + +BadEscape: + // Unrecognized escape sequence. + status->set_code(kRegexpBadEscape); + status->set_error_arg( + StringPiece(begin, static_cast(s->begin() - begin))); + return false; +} + +// Add a range to the character class, but exclude newline if asked. +// Also handle case folding. +void CharClassBuilder::AddRangeFlags( + Rune lo, Rune hi, Regexp::ParseFlags parse_flags) { + + // Take out \n if the flags say so. + bool cutnl = !(parse_flags & Regexp::ClassNL) || + (parse_flags & Regexp::NeverNL); + if (cutnl && lo <= '\n' && '\n' <= hi) { + if (lo < '\n') + AddRangeFlags(lo, '\n' - 1, parse_flags); + if (hi > '\n') + AddRangeFlags('\n' + 1, hi, parse_flags); + return; + } + + // If folding case, add fold-equivalent characters too. + if (parse_flags & Regexp::FoldCase) + AddFoldedRange(this, lo, hi, 0); + else + AddRange(lo, hi); +} + +// Look for a group with the given name. +static const UGroup* LookupGroup(const StringPiece& name, + const UGroup *groups, int ngroups) { + // Simple name lookup. + for (int i = 0; i < ngroups; i++) + if (StringPiece(groups[i].name) == name) + return &groups[i]; + return NULL; +} + +// Look for a POSIX group with the given name (e.g., "[:^alpha:]") +static const UGroup* LookupPosixGroup(const StringPiece& name) { + return LookupGroup(name, posix_groups, num_posix_groups); +} + +static const UGroup* LookupPerlGroup(const StringPiece& name) { + return LookupGroup(name, perl_groups, num_perl_groups); +} + +#if !defined(RE2_USE_ICU) +// Fake UGroup containing all Runes +static URange16 any16[] = { { 0, 65535 } }; +static URange32 any32[] = { { 65536, Runemax } }; +static UGroup anygroup = { "Any", +1, any16, 1, any32, 1 }; + +// Look for a Unicode group with the given name (e.g., "Han") +static const UGroup* LookupUnicodeGroup(const StringPiece& name) { + // Special case: "Any" means any. + if (name == StringPiece("Any")) + return &anygroup; + return LookupGroup(name, unicode_groups, num_unicode_groups); +} +#endif + +// Add a UGroup or its negation to the character class. +static void AddUGroup(CharClassBuilder *cc, const UGroup *g, int sign, + Regexp::ParseFlags parse_flags) { + if (sign == +1) { + for (int i = 0; i < g->nr16; i++) { + cc->AddRangeFlags(g->r16[i].lo, g->r16[i].hi, parse_flags); + } + for (int i = 0; i < g->nr32; i++) { + cc->AddRangeFlags(g->r32[i].lo, g->r32[i].hi, parse_flags); + } + } else { + if (parse_flags & Regexp::FoldCase) { + // Normally adding a case-folded group means + // adding all the extra fold-equivalent runes too. + // But if we're adding the negation of the group, + // we have to exclude all the runes that are fold-equivalent + // to what's already missing. Too hard, so do in two steps. + CharClassBuilder ccb1; + AddUGroup(&ccb1, g, +1, parse_flags); + // If the flags say to take out \n, put it in, so that negating will take it out. + // Normally AddRangeFlags does this, but we're bypassing AddRangeFlags. + bool cutnl = !(parse_flags & Regexp::ClassNL) || + (parse_flags & Regexp::NeverNL); + if (cutnl) { + ccb1.AddRange('\n', '\n'); + } + ccb1.Negate(); + cc->AddCharClass(&ccb1); + return; + } + int next = 0; + for (int i = 0; i < g->nr16; i++) { + if (next < g->r16[i].lo) + cc->AddRangeFlags(next, g->r16[i].lo - 1, parse_flags); + next = g->r16[i].hi + 1; + } + for (int i = 0; i < g->nr32; i++) { + if (next < g->r32[i].lo) + cc->AddRangeFlags(next, g->r32[i].lo - 1, parse_flags); + next = g->r32[i].hi + 1; + } + if (next <= Runemax) + cc->AddRangeFlags(next, Runemax, parse_flags); + } +} + +// Maybe parse a Perl character class escape sequence. +// Only recognizes the Perl character classes (\d \s \w \D \S \W), +// not the Perl empty-string classes (\b \B \A \Z \z). +// On success, sets *s to span the remainder of the string +// and returns the corresponding UGroup. +// The StringPiece must *NOT* be edited unless the call succeeds. +const UGroup* MaybeParsePerlCCEscape(StringPiece* s, Regexp::ParseFlags parse_flags) { + if (!(parse_flags & Regexp::PerlClasses)) + return NULL; + if (s->size() < 2 || (*s)[0] != '\\') + return NULL; + // Could use StringPieceToRune, but there aren't + // any non-ASCII Perl group names. + StringPiece name(s->begin(), 2); + const UGroup *g = LookupPerlGroup(name); + if (g == NULL) + return NULL; + s->remove_prefix(name.size()); + return g; +} + +enum ParseStatus { + kParseOk, // Did some parsing. + kParseError, // Found an error. + kParseNothing, // Decided not to parse. +}; + +// Maybe parses a Unicode character group like \p{Han} or \P{Han} +// (the latter is a negated group). +ParseStatus ParseUnicodeGroup(StringPiece* s, Regexp::ParseFlags parse_flags, + CharClassBuilder *cc, + RegexpStatus* status) { + // Decide whether to parse. + if (!(parse_flags & Regexp::UnicodeGroups)) + return kParseNothing; + if (s->size() < 2 || (*s)[0] != '\\') + return kParseNothing; + Rune c = (*s)[1]; + if (c != 'p' && c != 'P') + return kParseNothing; + + // Committed to parse. Results: + int sign = +1; // -1 = negated char class + if (c == 'P') + sign = -sign; + StringPiece seq = *s; // \p{Han} or \pL + StringPiece name; // Han or L + s->remove_prefix(2); // '\\', 'p' + + if (!StringPieceToRune(&c, s, status)) + return kParseError; + if (c != '{') { + // Name is the bit of string we just skipped over for c. + const char* p = seq.begin() + 2; + name = StringPiece(p, static_cast(s->begin() - p)); + } else { + // Name is in braces. Look for closing } + size_t end = s->find('}', 0); + if (end == StringPiece::npos) { + if (!IsValidUTF8(seq, status)) + return kParseError; + status->set_code(kRegexpBadCharRange); + status->set_error_arg(seq); + return kParseError; + } + name = StringPiece(s->begin(), end); // without '}' + s->remove_prefix(end + 1); // with '}' + if (!IsValidUTF8(name, status)) + return kParseError; + } + + // Chop seq where s now begins. + seq = StringPiece(seq.begin(), static_cast(s->begin() - seq.begin())); + + if (name.size() > 0 && name[0] == '^') { + sign = -sign; + name.remove_prefix(1); // '^' + } + +#if !defined(RE2_USE_ICU) + // Look up the group in the RE2 Unicode data. + const UGroup *g = LookupUnicodeGroup(name); + if (g == NULL) { + status->set_code(kRegexpBadCharRange); + status->set_error_arg(seq); + return kParseError; + } + + AddUGroup(cc, g, sign, parse_flags); +#else + // Look up the group in the ICU Unicode data. Because ICU provides full + // Unicode properties support, this could be more than a lookup by name. + ::icu::UnicodeString ustr = ::icu::UnicodeString::fromUTF8( + std::string("\\p{") + std::string(name) + std::string("}")); + UErrorCode uerr = U_ZERO_ERROR; + ::icu::UnicodeSet uset(ustr, uerr); + if (U_FAILURE(uerr)) { + status->set_code(kRegexpBadCharRange); + status->set_error_arg(seq); + return kParseError; + } + + // Convert the UnicodeSet to a URange32 and UGroup that we can add. + int nr = uset.getRangeCount(); + URange32* r = new URange32[nr]; + for (int i = 0; i < nr; i++) { + r[i].lo = uset.getRangeStart(i); + r[i].hi = uset.getRangeEnd(i); + } + UGroup g = {"", +1, 0, 0, r, nr}; + AddUGroup(cc, &g, sign, parse_flags); + delete[] r; +#endif + + return kParseOk; +} + +// Parses a character class name like [:alnum:]. +// Sets *s to span the remainder of the string. +// Adds the ranges corresponding to the class to ranges. +static ParseStatus ParseCCName(StringPiece* s, Regexp::ParseFlags parse_flags, + CharClassBuilder *cc, + RegexpStatus* status) { + // Check begins with [: + const char* p = s->data(); + const char* ep = s->data() + s->size(); + if (ep - p < 2 || p[0] != '[' || p[1] != ':') + return kParseNothing; + + // Look for closing :]. + const char* q; + for (q = p+2; q <= ep-2 && (*q != ':' || *(q+1) != ']'); q++) + ; + + // If no closing :], then ignore. + if (q > ep-2) + return kParseNothing; + + // Got it. Check that it's valid. + q += 2; + StringPiece name(p, static_cast(q - p)); + + const UGroup *g = LookupPosixGroup(name); + if (g == NULL) { + status->set_code(kRegexpBadCharRange); + status->set_error_arg(name); + return kParseError; + } + + s->remove_prefix(name.size()); + AddUGroup(cc, g, g->sign, parse_flags); + return kParseOk; +} + +// Parses a character inside a character class. +// There are fewer special characters here than in the rest of the regexp. +// Sets *s to span the remainder of the string. +// Sets *rp to the character. +bool Regexp::ParseState::ParseCCCharacter(StringPiece* s, Rune *rp, + const StringPiece& whole_class, + RegexpStatus* status) { + if (s->size() == 0) { + status->set_code(kRegexpMissingBracket); + status->set_error_arg(whole_class); + return false; + } + + // Allow regular escape sequences even though + // many need not be escaped in this context. + if (s->size() >= 1 && (*s)[0] == '\\') + return ParseEscape(s, rp, status, rune_max_); + + // Otherwise take the next rune. + return StringPieceToRune(rp, s, status) >= 0; +} + +// Parses a character class character, or, if the character +// is followed by a hyphen, parses a character class range. +// For single characters, rr->lo == rr->hi. +// Sets *s to span the remainder of the string. +// Sets *rp to the character. +bool Regexp::ParseState::ParseCCRange(StringPiece* s, RuneRange* rr, + const StringPiece& whole_class, + RegexpStatus* status) { + StringPiece os = *s; + if (!ParseCCCharacter(s, &rr->lo, whole_class, status)) + return false; + // [a-] means (a|-), so check for final ]. + if (s->size() >= 2 && (*s)[0] == '-' && (*s)[1] != ']') { + s->remove_prefix(1); // '-' + if (!ParseCCCharacter(s, &rr->hi, whole_class, status)) + return false; + if (rr->hi < rr->lo) { + status->set_code(kRegexpBadCharRange); + status->set_error_arg( + StringPiece(os.data(), static_cast(s->data() - os.data()))); + return false; + } + } else { + rr->hi = rr->lo; + } + return true; +} + +// Parses a possibly-negated character class expression like [^abx-z[:digit:]]. +// Sets *s to span the remainder of the string. +// Sets *out_re to the regexp for the class. +bool Regexp::ParseState::ParseCharClass(StringPiece* s, + Regexp** out_re, + RegexpStatus* status) { + StringPiece whole_class = *s; + if (s->size() == 0 || (*s)[0] != '[') { + // Caller checked this. + status->set_code(kRegexpInternalError); + status->set_error_arg(StringPiece()); + return false; + } + bool negated = false; + Regexp* re = new Regexp(kRegexpCharClass, flags_ & ~FoldCase); + re->ccb_ = new CharClassBuilder; + s->remove_prefix(1); // '[' + if (s->size() > 0 && (*s)[0] == '^') { + s->remove_prefix(1); // '^' + negated = true; + if (!(flags_ & ClassNL) || (flags_ & NeverNL)) { + // If NL can't match implicitly, then pretend + // negated classes include a leading \n. + re->ccb_->AddRange('\n', '\n'); + } + } + bool first = true; // ] is okay as first char in class + while (s->size() > 0 && ((*s)[0] != ']' || first)) { + // - is only okay unescaped as first or last in class. + // Except that Perl allows - anywhere. + if ((*s)[0] == '-' && !first && !(flags_&PerlX) && + (s->size() == 1 || (*s)[1] != ']')) { + StringPiece t = *s; + t.remove_prefix(1); // '-' + Rune r; + int n = StringPieceToRune(&r, &t, status); + if (n < 0) { + re->Decref(); + return false; + } + status->set_code(kRegexpBadCharRange); + status->set_error_arg(StringPiece(s->data(), 1+n)); + re->Decref(); + return false; + } + first = false; + + // Look for [:alnum:] etc. + if (s->size() > 2 && (*s)[0] == '[' && (*s)[1] == ':') { + switch (ParseCCName(s, flags_, re->ccb_, status)) { + case kParseOk: + continue; + case kParseError: + re->Decref(); + return false; + case kParseNothing: + break; + } + } + + // Look for Unicode character group like \p{Han} + if (s->size() > 2 && + (*s)[0] == '\\' && + ((*s)[1] == 'p' || (*s)[1] == 'P')) { + switch (ParseUnicodeGroup(s, flags_, re->ccb_, status)) { + case kParseOk: + continue; + case kParseError: + re->Decref(); + return false; + case kParseNothing: + break; + } + } + + // Look for Perl character class symbols (extension). + const UGroup *g = MaybeParsePerlCCEscape(s, flags_); + if (g != NULL) { + AddUGroup(re->ccb_, g, g->sign, flags_); + continue; + } + + // Otherwise assume single character or simple range. + RuneRange rr; + if (!ParseCCRange(s, &rr, whole_class, status)) { + re->Decref(); + return false; + } + // AddRangeFlags is usually called in response to a class like + // \p{Foo} or [[:foo:]]; for those, it filters \n out unless + // Regexp::ClassNL is set. In an explicit range or singleton + // like we just parsed, we do not filter \n out, so set ClassNL + // in the flags. + re->ccb_->AddRangeFlags(rr.lo, rr.hi, flags_ | Regexp::ClassNL); + } + if (s->size() == 0) { + status->set_code(kRegexpMissingBracket); + status->set_error_arg(whole_class); + re->Decref(); + return false; + } + s->remove_prefix(1); // ']' + + if (negated) + re->ccb_->Negate(); + + *out_re = re; + return true; +} + +// Is this a valid capture name? [A-Za-z0-9_]+ +// PCRE limits names to 32 bytes. +// Python rejects names starting with digits. +// We don't enforce either of those. +static bool IsValidCaptureName(const StringPiece& name) { + if (name.size() == 0) + return false; + for (size_t i = 0; i < name.size(); i++) { + int c = name[i]; + if (('0' <= c && c <= '9') || + ('a' <= c && c <= 'z') || + ('A' <= c && c <= 'Z') || + c == '_') + continue; + return false; + } + return true; +} + +// Parses a Perl flag setting or non-capturing group or both, +// like (?i) or (?: or (?i:. Removes from s, updates parse state. +// The caller must check that s begins with "(?". +// Returns true on success. If the Perl flag is not +// well-formed or not supported, sets status_ and returns false. +bool Regexp::ParseState::ParsePerlFlags(StringPiece* s) { + StringPiece t = *s; + + // Caller is supposed to check this. + if (!(flags_ & PerlX) || t.size() < 2 || t[0] != '(' || t[1] != '?') { + LOG(DFATAL) << "Bad call to ParseState::ParsePerlFlags"; + status_->set_code(kRegexpInternalError); + return false; + } + + t.remove_prefix(2); // "(?" + + // Check for named captures, first introduced in Python's regexp library. + // As usual, there are three slightly different syntaxes: + // + // (?Pexpr) the original, introduced by Python + // (?expr) the .NET alteration, adopted by Perl 5.10 + // (?'name'expr) another .NET alteration, adopted by Perl 5.10 + // + // Perl 5.10 gave in and implemented the Python version too, + // but they claim that the last two are the preferred forms. + // PCRE and languages based on it (specifically, PHP and Ruby) + // support all three as well. EcmaScript 4 uses only the Python form. + // + // In both the open source world (via Code Search) and the + // Google source tree, (?Pname) is the dominant form, + // so that's the one we implement. One is enough. + if (t.size() > 2 && t[0] == 'P' && t[1] == '<') { + // Pull out name. + size_t end = t.find('>', 2); + if (end == StringPiece::npos) { + if (!IsValidUTF8(*s, status_)) + return false; + status_->set_code(kRegexpBadNamedCapture); + status_->set_error_arg(*s); + return false; + } + + // t is "P...", t[end] == '>' + StringPiece capture(t.begin()-2, end+3); // "(?P" + StringPiece name(t.begin()+2, end-2); // "name" + if (!IsValidUTF8(name, status_)) + return false; + if (!IsValidCaptureName(name)) { + status_->set_code(kRegexpBadNamedCapture); + status_->set_error_arg(capture); + return false; + } + + if (!DoLeftParen(name)) { + // DoLeftParen's failure set status_. + return false; + } + + s->remove_prefix(static_cast(capture.end() - s->begin())); + return true; + } + + bool negated = false; + bool sawflags = false; + int nflags = flags_; + Rune c; + for (bool done = false; !done; ) { + if (t.size() == 0) + goto BadPerlOp; + if (StringPieceToRune(&c, &t, status_) < 0) + return false; + switch (c) { + default: + goto BadPerlOp; + + // Parse flags. + case 'i': + sawflags = true; + if (negated) + nflags &= ~FoldCase; + else + nflags |= FoldCase; + break; + + case 'm': // opposite of our OneLine + sawflags = true; + if (negated) + nflags |= OneLine; + else + nflags &= ~OneLine; + break; + + case 's': + sawflags = true; + if (negated) + nflags &= ~DotNL; + else + nflags |= DotNL; + break; + + case 'U': + sawflags = true; + if (negated) + nflags &= ~NonGreedy; + else + nflags |= NonGreedy; + break; + + // Negation + case '-': + if (negated) + goto BadPerlOp; + negated = true; + sawflags = false; + break; + + // Open new group. + case ':': + if (!DoLeftParenNoCapture()) { + // DoLeftParenNoCapture's failure set status_. + return false; + } + done = true; + break; + + // Finish flags. + case ')': + done = true; + break; + } + } + + if (negated && !sawflags) + goto BadPerlOp; + + flags_ = static_cast(nflags); + *s = t; + return true; + +BadPerlOp: + status_->set_code(kRegexpBadPerlOp); + status_->set_error_arg( + StringPiece(s->begin(), static_cast(t.begin() - s->begin()))); + return false; +} + +// Converts latin1 (assumed to be encoded as Latin1 bytes) +// into UTF8 encoding in string. +// Can't use EncodingUtils::EncodeLatin1AsUTF8 because it is +// deprecated and because it rejects code points 0x80-0x9F. +void ConvertLatin1ToUTF8(const StringPiece& latin1, std::string* utf) { + char buf[UTFmax]; + + utf->clear(); + for (size_t i = 0; i < latin1.size(); i++) { + Rune r = latin1[i] & 0xFF; + int n = runetochar(buf, &r); + utf->append(buf, n); + } +} + +// Parses the regular expression given by s, +// returning the corresponding Regexp tree. +// The caller must Decref the return value when done with it. +// Returns NULL on error. +Regexp* Regexp::Parse(const StringPiece& s, ParseFlags global_flags, + RegexpStatus* status) { + // Make status non-NULL (easier on everyone else). + RegexpStatus xstatus; + if (status == NULL) + status = &xstatus; + + ParseState ps(global_flags, s, status); + StringPiece t = s; + + // Convert regexp to UTF-8 (easier on the rest of the parser). + if (global_flags & Latin1) { + std::string* tmp = new std::string; + ConvertLatin1ToUTF8(t, tmp); + status->set_tmp(tmp); + t = *tmp; + } + + if (global_flags & Literal) { + // Special parse loop for literal string. + while (t.size() > 0) { + Rune r; + if (StringPieceToRune(&r, &t, status) < 0) + return NULL; + if (!ps.PushLiteral(r)) + return NULL; + } + return ps.DoFinish(); + } + + StringPiece lastunary = StringPiece(); + while (t.size() > 0) { + StringPiece isunary = StringPiece(); + switch (t[0]) { + default: { + Rune r; + if (StringPieceToRune(&r, &t, status) < 0) + return NULL; + if (!ps.PushLiteral(r)) + return NULL; + break; + } + + case '(': + // "(?" introduces Perl escape. + if ((ps.flags() & PerlX) && (t.size() >= 2 && t[1] == '?')) { + // Flag changes and non-capturing groups. + if (!ps.ParsePerlFlags(&t)) + return NULL; + break; + } + if (ps.flags() & NeverCapture) { + if (!ps.DoLeftParenNoCapture()) + return NULL; + } else { + if (!ps.DoLeftParen(StringPiece())) + return NULL; + } + t.remove_prefix(1); // '(' + break; + + case '|': + if (!ps.DoVerticalBar()) + return NULL; + t.remove_prefix(1); // '|' + break; + + case ')': + if (!ps.DoRightParen()) + return NULL; + t.remove_prefix(1); // ')' + break; + + case '^': // Beginning of line. + if (!ps.PushCarat()) + return NULL; + t.remove_prefix(1); // '^' + break; + + case '$': // End of line. + if (!ps.PushDollar()) + return NULL; + t.remove_prefix(1); // '$' + break; + + case '.': // Any character (possibly except newline). + if (!ps.PushDot()) + return NULL; + t.remove_prefix(1); // '.' + break; + + case '[': { // Character class. + Regexp* re; + if (!ps.ParseCharClass(&t, &re, status)) + return NULL; + if (!ps.PushRegexp(re)) + return NULL; + break; + } + + case '*': { // Zero or more. + RegexpOp op; + op = kRegexpStar; + goto Rep; + case '+': // One or more. + op = kRegexpPlus; + goto Rep; + case '?': // Zero or one. + op = kRegexpQuest; + goto Rep; + Rep: + StringPiece opstr = t; + bool nongreedy = false; + t.remove_prefix(1); // '*' or '+' or '?' + if (ps.flags() & PerlX) { + if (t.size() > 0 && t[0] == '?') { + nongreedy = true; + t.remove_prefix(1); // '?' + } + if (lastunary.size() > 0) { + // In Perl it is not allowed to stack repetition operators: + // a** is a syntax error, not a double-star. + // (and a++ means something else entirely, which we don't support!) + status->set_code(kRegexpRepeatOp); + status->set_error_arg(StringPiece( + lastunary.begin(), + static_cast(t.begin() - lastunary.begin()))); + return NULL; + } + } + opstr = StringPiece(opstr.data(), + static_cast(t.data() - opstr.data())); + if (!ps.PushRepeatOp(op, opstr, nongreedy)) + return NULL; + isunary = opstr; + break; + } + + case '{': { // Counted repetition. + int lo, hi; + StringPiece opstr = t; + if (!MaybeParseRepetition(&t, &lo, &hi)) { + // Treat like a literal. + if (!ps.PushLiteral('{')) + return NULL; + t.remove_prefix(1); // '{' + break; + } + bool nongreedy = false; + if (ps.flags() & PerlX) { + if (t.size() > 0 && t[0] == '?') { + nongreedy = true; + t.remove_prefix(1); // '?' + } + if (lastunary.size() > 0) { + // Not allowed to stack repetition operators. + status->set_code(kRegexpRepeatOp); + status->set_error_arg(StringPiece( + lastunary.begin(), + static_cast(t.begin() - lastunary.begin()))); + return NULL; + } + } + opstr = StringPiece(opstr.data(), + static_cast(t.data() - opstr.data())); + if (!ps.PushRepetition(lo, hi, opstr, nongreedy)) + return NULL; + isunary = opstr; + break; + } + + case '\\': { // Escaped character or Perl sequence. + // \b and \B: word boundary or not + if ((ps.flags() & Regexp::PerlB) && + t.size() >= 2 && (t[1] == 'b' || t[1] == 'B')) { + if (!ps.PushWordBoundary(t[1] == 'b')) + return NULL; + t.remove_prefix(2); // '\\', 'b' + break; + } + + if ((ps.flags() & Regexp::PerlX) && t.size() >= 2) { + if (t[1] == 'A') { + if (!ps.PushSimpleOp(kRegexpBeginText)) + return NULL; + t.remove_prefix(2); // '\\', 'A' + break; + } + if (t[1] == 'z') { + if (!ps.PushSimpleOp(kRegexpEndText)) + return NULL; + t.remove_prefix(2); // '\\', 'z' + break; + } + // Do not recognize \Z, because this library can't + // implement the exact Perl/PCRE semantics. + // (This library treats "(?-m)$" as \z, even though + // in Perl and PCRE it is equivalent to \Z.) + + if (t[1] == 'C') { // \C: any byte [sic] + if (!ps.PushSimpleOp(kRegexpAnyByte)) + return NULL; + t.remove_prefix(2); // '\\', 'C' + break; + } + + if (t[1] == 'Q') { // \Q ... \E: the ... is always literals + t.remove_prefix(2); // '\\', 'Q' + while (t.size() > 0) { + if (t.size() >= 2 && t[0] == '\\' && t[1] == 'E') { + t.remove_prefix(2); // '\\', 'E' + break; + } + Rune r; + if (StringPieceToRune(&r, &t, status) < 0) + return NULL; + if (!ps.PushLiteral(r)) + return NULL; + } + break; + } + } + + if (t.size() >= 2 && (t[1] == 'p' || t[1] == 'P')) { + Regexp* re = new Regexp(kRegexpCharClass, ps.flags() & ~FoldCase); + re->ccb_ = new CharClassBuilder; + switch (ParseUnicodeGroup(&t, ps.flags(), re->ccb_, status)) { + case kParseOk: + if (!ps.PushRegexp(re)) + return NULL; + goto Break2; + case kParseError: + re->Decref(); + return NULL; + case kParseNothing: + re->Decref(); + break; + } + } + + const UGroup *g = MaybeParsePerlCCEscape(&t, ps.flags()); + if (g != NULL) { + Regexp* re = new Regexp(kRegexpCharClass, ps.flags() & ~FoldCase); + re->ccb_ = new CharClassBuilder; + AddUGroup(re->ccb_, g, g->sign, ps.flags()); + if (!ps.PushRegexp(re)) + return NULL; + break; + } + + Rune r; + if (!ParseEscape(&t, &r, status, ps.rune_max())) + return NULL; + if (!ps.PushLiteral(r)) + return NULL; + break; + } + } + Break2: + lastunary = isunary; + } + return ps.DoFinish(); +} + +} // namespace re2 diff --git a/extern/re2/re2/perl_groups.cc b/extern/re2/re2/perl_groups.cc new file mode 100644 index 0000000000..422b3882d4 --- /dev/null +++ b/extern/re2/re2/perl_groups.cc @@ -0,0 +1,119 @@ +// GENERATED BY make_perl_groups.pl; DO NOT EDIT. +// make_perl_groups.pl >perl_groups.cc + +#include "re2/unicode_groups.h" + +namespace re2 { + +static const URange16 code1[] = { /* \d */ + { 0x30, 0x39 }, +}; +static const URange16 code2[] = { /* \s */ + { 0x9, 0xa }, + { 0xc, 0xd }, + { 0x20, 0x20 }, +}; +static const URange16 code3[] = { /* \w */ + { 0x30, 0x39 }, + { 0x41, 0x5a }, + { 0x5f, 0x5f }, + { 0x61, 0x7a }, +}; +const UGroup perl_groups[] = { + { "\\d", +1, code1, 1 }, + { "\\D", -1, code1, 1 }, + { "\\s", +1, code2, 3 }, + { "\\S", -1, code2, 3 }, + { "\\w", +1, code3, 4 }, + { "\\W", -1, code3, 4 }, +}; +const int num_perl_groups = 6; +static const URange16 code4[] = { /* [:alnum:] */ + { 0x30, 0x39 }, + { 0x41, 0x5a }, + { 0x61, 0x7a }, +}; +static const URange16 code5[] = { /* [:alpha:] */ + { 0x41, 0x5a }, + { 0x61, 0x7a }, +}; +static const URange16 code6[] = { /* [:ascii:] */ + { 0x0, 0x7f }, +}; +static const URange16 code7[] = { /* [:blank:] */ + { 0x9, 0x9 }, + { 0x20, 0x20 }, +}; +static const URange16 code8[] = { /* [:cntrl:] */ + { 0x0, 0x1f }, + { 0x7f, 0x7f }, +}; +static const URange16 code9[] = { /* [:digit:] */ + { 0x30, 0x39 }, +}; +static const URange16 code10[] = { /* [:graph:] */ + { 0x21, 0x7e }, +}; +static const URange16 code11[] = { /* [:lower:] */ + { 0x61, 0x7a }, +}; +static const URange16 code12[] = { /* [:print:] */ + { 0x20, 0x7e }, +}; +static const URange16 code13[] = { /* [:punct:] */ + { 0x21, 0x2f }, + { 0x3a, 0x40 }, + { 0x5b, 0x60 }, + { 0x7b, 0x7e }, +}; +static const URange16 code14[] = { /* [:space:] */ + { 0x9, 0xd }, + { 0x20, 0x20 }, +}; +static const URange16 code15[] = { /* [:upper:] */ + { 0x41, 0x5a }, +}; +static const URange16 code16[] = { /* [:word:] */ + { 0x30, 0x39 }, + { 0x41, 0x5a }, + { 0x5f, 0x5f }, + { 0x61, 0x7a }, +}; +static const URange16 code17[] = { /* [:xdigit:] */ + { 0x30, 0x39 }, + { 0x41, 0x46 }, + { 0x61, 0x66 }, +}; +const UGroup posix_groups[] = { + { "[:alnum:]", +1, code4, 3 }, + { "[:^alnum:]", -1, code4, 3 }, + { "[:alpha:]", +1, code5, 2 }, + { "[:^alpha:]", -1, code5, 2 }, + { "[:ascii:]", +1, code6, 1 }, + { "[:^ascii:]", -1, code6, 1 }, + { "[:blank:]", +1, code7, 2 }, + { "[:^blank:]", -1, code7, 2 }, + { "[:cntrl:]", +1, code8, 2 }, + { "[:^cntrl:]", -1, code8, 2 }, + { "[:digit:]", +1, code9, 1 }, + { "[:^digit:]", -1, code9, 1 }, + { "[:graph:]", +1, code10, 1 }, + { "[:^graph:]", -1, code10, 1 }, + { "[:lower:]", +1, code11, 1 }, + { "[:^lower:]", -1, code11, 1 }, + { "[:print:]", +1, code12, 1 }, + { "[:^print:]", -1, code12, 1 }, + { "[:punct:]", +1, code13, 4 }, + { "[:^punct:]", -1, code13, 4 }, + { "[:space:]", +1, code14, 2 }, + { "[:^space:]", -1, code14, 2 }, + { "[:upper:]", +1, code15, 1 }, + { "[:^upper:]", -1, code15, 1 }, + { "[:word:]", +1, code16, 4 }, + { "[:^word:]", -1, code16, 4 }, + { "[:xdigit:]", +1, code17, 3 }, + { "[:^xdigit:]", -1, code17, 3 }, +}; +const int num_posix_groups = 28; + +} // namespace re2 diff --git a/extern/re2/re2/prefilter.cc b/extern/re2/re2/prefilter.cc new file mode 100644 index 0000000000..f61d54b8f8 --- /dev/null +++ b/extern/re2/re2/prefilter.cc @@ -0,0 +1,710 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "re2/prefilter.h" + +#include +#include +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "util/strutil.h" +#include "util/utf.h" +#include "re2/re2.h" +#include "re2/unicode_casefold.h" +#include "re2/walker-inl.h" + +namespace re2 { + +static const bool ExtraDebug = false; + +typedef std::set::iterator SSIter; +typedef std::set::const_iterator ConstSSIter; + +// Initializes a Prefilter, allocating subs_ as necessary. +Prefilter::Prefilter(Op op) { + op_ = op; + subs_ = NULL; + if (op_ == AND || op_ == OR) + subs_ = new std::vector; +} + +// Destroys a Prefilter. +Prefilter::~Prefilter() { + if (subs_) { + for (size_t i = 0; i < subs_->size(); i++) + delete (*subs_)[i]; + delete subs_; + subs_ = NULL; + } +} + +// Simplify if the node is an empty Or or And. +Prefilter* Prefilter::Simplify() { + if (op_ != AND && op_ != OR) { + return this; + } + + // Nothing left in the AND/OR. + if (subs_->empty()) { + if (op_ == AND) + op_ = ALL; // AND of nothing is true + else + op_ = NONE; // OR of nothing is false + + return this; + } + + // Just one subnode: throw away wrapper. + if (subs_->size() == 1) { + Prefilter* a = (*subs_)[0]; + subs_->clear(); + delete this; + return a->Simplify(); + } + + return this; +} + +// Combines two Prefilters together to create an "op" (AND or OR). +// The passed Prefilters will be part of the returned Prefilter or deleted. +// Does lots of work to avoid creating unnecessarily complicated structures. +Prefilter* Prefilter::AndOr(Op op, Prefilter* a, Prefilter* b) { + // If a, b can be rewritten as op, do so. + a = a->Simplify(); + b = b->Simplify(); + + // Canonicalize: a->op <= b->op. + if (a->op() > b->op()) { + Prefilter* t = a; + a = b; + b = t; + } + + // Trivial cases. + // ALL AND b = b + // NONE OR b = b + // ALL OR b = ALL + // NONE AND b = NONE + // Don't need to look at b, because of canonicalization above. + // ALL and NONE are smallest opcodes. + if (a->op() == ALL || a->op() == NONE) { + if ((a->op() == ALL && op == AND) || + (a->op() == NONE && op == OR)) { + delete a; + return b; + } else { + delete b; + return a; + } + } + + // If a and b match op, merge their contents. + if (a->op() == op && b->op() == op) { + for (size_t i = 0; i < b->subs()->size(); i++) { + Prefilter* bb = (*b->subs())[i]; + a->subs()->push_back(bb); + } + b->subs()->clear(); + delete b; + return a; + } + + // If a already has the same op as the op that is under construction + // add in b (similarly if b already has the same op, add in a). + if (b->op() == op) { + Prefilter* t = a; + a = b; + b = t; + } + if (a->op() == op) { + a->subs()->push_back(b); + return a; + } + + // Otherwise just return the op. + Prefilter* c = new Prefilter(op); + c->subs()->push_back(a); + c->subs()->push_back(b); + return c; +} + +Prefilter* Prefilter::And(Prefilter* a, Prefilter* b) { + return AndOr(AND, a, b); +} + +Prefilter* Prefilter::Or(Prefilter* a, Prefilter* b) { + return AndOr(OR, a, b); +} + +static void SimplifyStringSet(std::set* ss) { + // Now make sure that the strings aren't redundant. For example, if + // we know "ab" is a required string, then it doesn't help at all to + // know that "abc" is also a required string, so delete "abc". This + // is because, when we are performing a string search to filter + // regexps, matching "ab" will already allow this regexp to be a + // candidate for match, so further matching "abc" is redundant. + // Note that we must ignore "" because find() would find it at the + // start of everything and thus we would end up erasing everything. + for (SSIter i = ss->begin(); i != ss->end(); ++i) { + if (i->empty()) + continue; + SSIter j = i; + ++j; + while (j != ss->end()) { + if (j->find(*i) != std::string::npos) { + j = ss->erase(j); + continue; + } + ++j; + } + } +} + +Prefilter* Prefilter::OrStrings(std::set* ss) { + Prefilter* or_prefilter = new Prefilter(NONE); + SimplifyStringSet(ss); + for (SSIter i = ss->begin(); i != ss->end(); ++i) + or_prefilter = Or(or_prefilter, FromString(*i)); + return or_prefilter; +} + +static Rune ToLowerRune(Rune r) { + if (r < Runeself) { + if ('A' <= r && r <= 'Z') + r += 'a' - 'A'; + return r; + } + + const CaseFold *f = LookupCaseFold(unicode_tolower, num_unicode_tolower, r); + if (f == NULL || r < f->lo) + return r; + return ApplyFold(f, r); +} + +static Rune ToLowerRuneLatin1(Rune r) { + if ('A' <= r && r <= 'Z') + r += 'a' - 'A'; + return r; +} + +Prefilter* Prefilter::FromString(const std::string& str) { + Prefilter* m = new Prefilter(Prefilter::ATOM); + m->atom_ = str; + return m; +} + +// Information about a regexp used during computation of Prefilter. +// Can be thought of as information about the set of strings matching +// the given regular expression. +class Prefilter::Info { + public: + Info(); + ~Info(); + + // More constructors. They delete their Info* arguments. + static Info* Alt(Info* a, Info* b); + static Info* Concat(Info* a, Info* b); + static Info* And(Info* a, Info* b); + static Info* Star(Info* a); + static Info* Plus(Info* a); + static Info* Quest(Info* a); + static Info* EmptyString(); + static Info* NoMatch(); + static Info* AnyCharOrAnyByte(); + static Info* CClass(CharClass* cc, bool latin1); + static Info* Literal(Rune r); + static Info* LiteralLatin1(Rune r); + static Info* AnyMatch(); + + // Format Info as a string. + std::string ToString(); + + // Caller takes ownership of the Prefilter. + Prefilter* TakeMatch(); + + std::set& exact() { return exact_; } + + bool is_exact() const { return is_exact_; } + + class Walker; + + private: + std::set exact_; + + // When is_exact_ is true, the strings that match + // are placed in exact_. When it is no longer an exact + // set of strings that match this RE, then is_exact_ + // is false and the match_ contains the required match + // criteria. + bool is_exact_; + + // Accumulated Prefilter query that any + // match for this regexp is guaranteed to match. + Prefilter* match_; +}; + + +Prefilter::Info::Info() + : is_exact_(false), + match_(NULL) { +} + +Prefilter::Info::~Info() { + delete match_; +} + +Prefilter* Prefilter::Info::TakeMatch() { + if (is_exact_) { + match_ = Prefilter::OrStrings(&exact_); + is_exact_ = false; + } + Prefilter* m = match_; + match_ = NULL; + return m; +} + +// Format a Info in string form. +std::string Prefilter::Info::ToString() { + if (is_exact_) { + int n = 0; + std::string s; + for (SSIter i = exact_.begin(); i != exact_.end(); ++i) { + if (n++ > 0) + s += ","; + s += *i; + } + return s; + } + + if (match_) + return match_->DebugString(); + + return ""; +} + +// Add the strings from src to dst. +static void CopyIn(const std::set& src, + std::set* dst) { + for (ConstSSIter i = src.begin(); i != src.end(); ++i) + dst->insert(*i); +} + +// Add the cross-product of a and b to dst. +// (For each string i in a and j in b, add i+j.) +static void CrossProduct(const std::set& a, + const std::set& b, + std::set* dst) { + for (ConstSSIter i = a.begin(); i != a.end(); ++i) + for (ConstSSIter j = b.begin(); j != b.end(); ++j) + dst->insert(*i + *j); +} + +// Concats a and b. Requires that both are exact sets. +// Forms an exact set that is a crossproduct of a and b. +Prefilter::Info* Prefilter::Info::Concat(Info* a, Info* b) { + if (a == NULL) + return b; + DCHECK(a->is_exact_); + DCHECK(b && b->is_exact_); + Info *ab = new Info(); + + CrossProduct(a->exact_, b->exact_, &ab->exact_); + ab->is_exact_ = true; + + delete a; + delete b; + return ab; +} + +// Constructs an inexact Info for ab given a and b. +// Used only when a or b is not exact or when the +// exact cross product is likely to be too big. +Prefilter::Info* Prefilter::Info::And(Info* a, Info* b) { + if (a == NULL) + return b; + if (b == NULL) + return a; + + Info *ab = new Info(); + + ab->match_ = Prefilter::And(a->TakeMatch(), b->TakeMatch()); + ab->is_exact_ = false; + delete a; + delete b; + return ab; +} + +// Constructs Info for a|b given a and b. +Prefilter::Info* Prefilter::Info::Alt(Info* a, Info* b) { + Info *ab = new Info(); + + if (a->is_exact_ && b->is_exact_) { + CopyIn(a->exact_, &ab->exact_); + CopyIn(b->exact_, &ab->exact_); + ab->is_exact_ = true; + } else { + // Either a or b has is_exact_ = false. If the other + // one has is_exact_ = true, we move it to match_ and + // then create a OR of a,b. The resulting Info has + // is_exact_ = false. + ab->match_ = Prefilter::Or(a->TakeMatch(), b->TakeMatch()); + ab->is_exact_ = false; + } + + delete a; + delete b; + return ab; +} + +// Constructs Info for a? given a. +Prefilter::Info* Prefilter::Info::Quest(Info *a) { + Info *ab = new Info(); + + ab->is_exact_ = false; + ab->match_ = new Prefilter(ALL); + delete a; + return ab; +} + +// Constructs Info for a* given a. +// Same as a? -- not much to do. +Prefilter::Info* Prefilter::Info::Star(Info *a) { + return Quest(a); +} + +// Constructs Info for a+ given a. If a was exact set, it isn't +// anymore. +Prefilter::Info* Prefilter::Info::Plus(Info *a) { + Info *ab = new Info(); + + ab->match_ = a->TakeMatch(); + ab->is_exact_ = false; + + delete a; + return ab; +} + +static std::string RuneToString(Rune r) { + char buf[UTFmax]; + int n = runetochar(buf, &r); + return std::string(buf, n); +} + +static std::string RuneToStringLatin1(Rune r) { + char c = r & 0xff; + return std::string(&c, 1); +} + +// Constructs Info for literal rune. +Prefilter::Info* Prefilter::Info::Literal(Rune r) { + Info* info = new Info(); + info->exact_.insert(RuneToString(ToLowerRune(r))); + info->is_exact_ = true; + return info; +} + +// Constructs Info for literal rune for Latin1 encoded string. +Prefilter::Info* Prefilter::Info::LiteralLatin1(Rune r) { + Info* info = new Info(); + info->exact_.insert(RuneToStringLatin1(ToLowerRuneLatin1(r))); + info->is_exact_ = true; + return info; +} + +// Constructs Info for dot (any character) or \C (any byte). +Prefilter::Info* Prefilter::Info::AnyCharOrAnyByte() { + Prefilter::Info* info = new Prefilter::Info(); + info->match_ = new Prefilter(ALL); + return info; +} + +// Constructs Prefilter::Info for no possible match. +Prefilter::Info* Prefilter::Info::NoMatch() { + Prefilter::Info* info = new Prefilter::Info(); + info->match_ = new Prefilter(NONE); + return info; +} + +// Constructs Prefilter::Info for any possible match. +// This Prefilter::Info is valid for any regular expression, +// since it makes no assertions whatsoever about the +// strings being matched. +Prefilter::Info* Prefilter::Info::AnyMatch() { + Prefilter::Info *info = new Prefilter::Info(); + info->match_ = new Prefilter(ALL); + return info; +} + +// Constructs Prefilter::Info for just the empty string. +Prefilter::Info* Prefilter::Info::EmptyString() { + Prefilter::Info* info = new Prefilter::Info(); + info->is_exact_ = true; + info->exact_.insert(""); + return info; +} + +// Constructs Prefilter::Info for a character class. +typedef CharClass::iterator CCIter; +Prefilter::Info* Prefilter::Info::CClass(CharClass *cc, + bool latin1) { + if (ExtraDebug) { + LOG(ERROR) << "CharClassInfo:"; + for (CCIter i = cc->begin(); i != cc->end(); ++i) + LOG(ERROR) << " " << i->lo << "-" << i->hi; + } + + // If the class is too large, it's okay to overestimate. + if (cc->size() > 10) + return AnyCharOrAnyByte(); + + Prefilter::Info *a = new Prefilter::Info(); + for (CCIter i = cc->begin(); i != cc->end(); ++i) + for (Rune r = i->lo; r <= i->hi; r++) { + if (latin1) { + a->exact_.insert(RuneToStringLatin1(ToLowerRuneLatin1(r))); + } else { + a->exact_.insert(RuneToString(ToLowerRune(r))); + } + } + + + a->is_exact_ = true; + + if (ExtraDebug) + LOG(ERROR) << " = " << a->ToString(); + + return a; +} + +class Prefilter::Info::Walker : public Regexp::Walker { + public: + Walker(bool latin1) : latin1_(latin1) {} + + virtual Info* PostVisit( + Regexp* re, Info* parent_arg, + Info* pre_arg, + Info** child_args, int nchild_args); + + virtual Info* ShortVisit( + Regexp* re, + Info* parent_arg); + + bool latin1() { return latin1_; } + private: + bool latin1_; + + Walker(const Walker&) = delete; + Walker& operator=(const Walker&) = delete; +}; + +Prefilter::Info* Prefilter::BuildInfo(Regexp* re) { + if (ExtraDebug) + LOG(ERROR) << "BuildPrefilter::Info: " << re->ToString(); + + bool latin1 = (re->parse_flags() & Regexp::Latin1) != 0; + Prefilter::Info::Walker w(latin1); + Prefilter::Info* info = w.WalkExponential(re, NULL, 100000); + + if (w.stopped_early()) { + delete info; + return NULL; + } + + return info; +} + +Prefilter::Info* Prefilter::Info::Walker::ShortVisit( + Regexp* re, Prefilter::Info* parent_arg) { + return AnyMatch(); +} + +// Constructs the Prefilter::Info for the given regular expression. +// Assumes re is simplified. +Prefilter::Info* Prefilter::Info::Walker::PostVisit( + Regexp* re, Prefilter::Info* parent_arg, + Prefilter::Info* pre_arg, Prefilter::Info** child_args, + int nchild_args) { + Prefilter::Info *info; + switch (re->op()) { + default: + case kRegexpRepeat: + LOG(DFATAL) << "Bad regexp op " << re->op(); + info = EmptyString(); + break; + + case kRegexpNoMatch: + info = NoMatch(); + break; + + // These ops match the empty string: + case kRegexpEmptyMatch: // anywhere + case kRegexpBeginLine: // at beginning of line + case kRegexpEndLine: // at end of line + case kRegexpBeginText: // at beginning of text + case kRegexpEndText: // at end of text + case kRegexpWordBoundary: // at word boundary + case kRegexpNoWordBoundary: // not at word boundary + info = EmptyString(); + break; + + case kRegexpLiteral: + if (latin1()) { + info = LiteralLatin1(re->rune()); + } + else { + info = Literal(re->rune()); + } + break; + + case kRegexpLiteralString: + if (re->nrunes() == 0) { + info = NoMatch(); + break; + } + if (latin1()) { + info = LiteralLatin1(re->runes()[0]); + for (int i = 1; i < re->nrunes(); i++) { + info = Concat(info, LiteralLatin1(re->runes()[i])); + } + } else { + info = Literal(re->runes()[0]); + for (int i = 1; i < re->nrunes(); i++) { + info = Concat(info, Literal(re->runes()[i])); + } + } + break; + + case kRegexpConcat: { + // Accumulate in info. + // Exact is concat of recent contiguous exact nodes. + info = NULL; + Info* exact = NULL; + for (int i = 0; i < nchild_args; i++) { + Info* ci = child_args[i]; // child info + if (!ci->is_exact() || + (exact && ci->exact().size() * exact->exact().size() > 16)) { + // Exact run is over. + info = And(info, exact); + exact = NULL; + // Add this child's info. + info = And(info, ci); + } else { + // Append to exact run. + exact = Concat(exact, ci); + } + } + info = And(info, exact); + } + break; + + case kRegexpAlternate: + info = child_args[0]; + for (int i = 1; i < nchild_args; i++) + info = Alt(info, child_args[i]); + break; + + case kRegexpStar: + info = Star(child_args[0]); + break; + + case kRegexpQuest: + info = Quest(child_args[0]); + break; + + case kRegexpPlus: + info = Plus(child_args[0]); + break; + + case kRegexpAnyChar: + case kRegexpAnyByte: + // Claim nothing, except that it's not empty. + info = AnyCharOrAnyByte(); + break; + + case kRegexpCharClass: + info = CClass(re->cc(), latin1()); + break; + + case kRegexpCapture: + // These don't affect the set of matching strings. + info = child_args[0]; + break; + } + + if (ExtraDebug) + LOG(ERROR) << "BuildInfo " << re->ToString() + << ": " << (info ? info->ToString() : ""); + + return info; +} + + +Prefilter* Prefilter::FromRegexp(Regexp* re) { + if (re == NULL) + return NULL; + + Regexp* simple = re->Simplify(); + Prefilter::Info *info = BuildInfo(simple); + + simple->Decref(); + if (info == NULL) + return NULL; + + Prefilter* m = info->TakeMatch(); + + delete info; + return m; +} + +std::string Prefilter::DebugString() const { + switch (op_) { + default: + LOG(DFATAL) << "Bad op in Prefilter::DebugString: " << op_; + return StringPrintf("op%d", op_); + case NONE: + return "*no-matches*"; + case ATOM: + return atom_; + case ALL: + return ""; + case AND: { + std::string s = ""; + for (size_t i = 0; i < subs_->size(); i++) { + if (i > 0) + s += " "; + Prefilter* sub = (*subs_)[i]; + s += sub ? sub->DebugString() : ""; + } + return s; + } + case OR: { + std::string s = "("; + for (size_t i = 0; i < subs_->size(); i++) { + if (i > 0) + s += "|"; + Prefilter* sub = (*subs_)[i]; + s += sub ? sub->DebugString() : ""; + } + s += ")"; + return s; + } + } +} + +Prefilter* Prefilter::FromRE2(const RE2* re2) { + if (re2 == NULL) + return NULL; + + Regexp* regexp = re2->Regexp(); + if (regexp == NULL) + return NULL; + + return FromRegexp(regexp); +} + + +} // namespace re2 diff --git a/extern/re2/re2/prefilter.h b/extern/re2/re2/prefilter.h new file mode 100644 index 0000000000..4fedeb4a7c --- /dev/null +++ b/extern/re2/re2/prefilter.h @@ -0,0 +1,108 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_PREFILTER_H_ +#define RE2_PREFILTER_H_ + +// Prefilter is the class used to extract string guards from regexps. +// Rather than using Prefilter class directly, use FilteredRE2. +// See filtered_re2.h + +#include +#include +#include + +#include "util/util.h" +#include "util/logging.h" + +namespace re2 { + +class RE2; + +class Regexp; + +class Prefilter { + // Instead of using Prefilter directly, use FilteredRE2; see filtered_re2.h + public: + enum Op { + ALL = 0, // Everything matches + NONE, // Nothing matches + ATOM, // The string atom() must match + AND, // All in subs() must match + OR, // One of subs() must match + }; + + explicit Prefilter(Op op); + ~Prefilter(); + + Op op() { return op_; } + const std::string& atom() const { return atom_; } + void set_unique_id(int id) { unique_id_ = id; } + int unique_id() const { return unique_id_; } + + // The children of the Prefilter node. + std::vector* subs() { + DCHECK(op_ == AND || op_ == OR); + return subs_; + } + + // Set the children vector. Prefilter takes ownership of subs and + // subs_ will be deleted when Prefilter is deleted. + void set_subs(std::vector* subs) { subs_ = subs; } + + // Given a RE2, return a Prefilter. The caller takes ownership of + // the Prefilter and should deallocate it. Returns NULL if Prefilter + // cannot be formed. + static Prefilter* FromRE2(const RE2* re2); + + // Returns a readable debug string of the prefilter. + std::string DebugString() const; + + private: + class Info; + + // Combines two prefilters together to create an AND. The passed + // Prefilters will be part of the returned Prefilter or deleted. + static Prefilter* And(Prefilter* a, Prefilter* b); + + // Combines two prefilters together to create an OR. The passed + // Prefilters will be part of the returned Prefilter or deleted. + static Prefilter* Or(Prefilter* a, Prefilter* b); + + // Generalized And/Or + static Prefilter* AndOr(Op op, Prefilter* a, Prefilter* b); + + static Prefilter* FromRegexp(Regexp* a); + + static Prefilter* FromString(const std::string& str); + + static Prefilter* OrStrings(std::set* ss); + + static Info* BuildInfo(Regexp* re); + + Prefilter* Simplify(); + + // Kind of Prefilter. + Op op_; + + // Sub-matches for AND or OR Prefilter. + std::vector* subs_; + + // Actual string to match in leaf node. + std::string atom_; + + // If different prefilters have the same string atom, or if they are + // structurally the same (e.g., OR of same atom strings) they are + // considered the same unique nodes. This is the id for each unique + // node. This field is populated with a unique id for every node, + // and -1 for duplicate nodes. + int unique_id_; + + Prefilter(const Prefilter&) = delete; + Prefilter& operator=(const Prefilter&) = delete; +}; + +} // namespace re2 + +#endif // RE2_PREFILTER_H_ diff --git a/extern/re2/re2/prefilter_tree.cc b/extern/re2/re2/prefilter_tree.cc new file mode 100644 index 0000000000..187e2ec552 --- /dev/null +++ b/extern/re2/re2/prefilter_tree.cc @@ -0,0 +1,407 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "re2/prefilter_tree.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "util/strutil.h" +#include "re2/prefilter.h" +#include "re2/re2.h" + +namespace re2 { + +static const bool ExtraDebug = false; + +PrefilterTree::PrefilterTree() + : compiled_(false), + min_atom_len_(3) { +} + +PrefilterTree::PrefilterTree(int min_atom_len) + : compiled_(false), + min_atom_len_(min_atom_len) { +} + +PrefilterTree::~PrefilterTree() { + for (size_t i = 0; i < prefilter_vec_.size(); i++) + delete prefilter_vec_[i]; + + for (size_t i = 0; i < entries_.size(); i++) + delete entries_[i].parents; +} + +void PrefilterTree::Add(Prefilter* prefilter) { + if (compiled_) { + LOG(DFATAL) << "Add called after Compile."; + return; + } + if (prefilter != NULL && !KeepNode(prefilter)) { + delete prefilter; + prefilter = NULL; + } + + prefilter_vec_.push_back(prefilter); +} + +void PrefilterTree::Compile(std::vector* atom_vec) { + if (compiled_) { + LOG(DFATAL) << "Compile called already."; + return; + } + + // Some legacy users of PrefilterTree call Compile() before + // adding any regexps and expect Compile() to have no effect. + if (prefilter_vec_.empty()) + return; + + compiled_ = true; + + // TODO(junyer): Use std::unordered_set instead? + NodeMap nodes; + AssignUniqueIds(&nodes, atom_vec); + + // Identify nodes that are too common among prefilters and are + // triggering too many parents. Then get rid of them if possible. + // Note that getting rid of a prefilter node simply means they are + // no longer necessary for their parent to trigger; that is, we do + // not miss out on any regexps triggering by getting rid of a + // prefilter node. + for (size_t i = 0; i < entries_.size(); i++) { + StdIntMap* parents = entries_[i].parents; + if (parents->size() > 8) { + // This one triggers too many things. If all the parents are AND + // nodes and have other things guarding them, then get rid of + // this trigger. TODO(vsri): Adjust the threshold appropriately, + // make it a function of total number of nodes? + bool have_other_guard = true; + for (StdIntMap::iterator it = parents->begin(); + it != parents->end(); ++it) { + have_other_guard = have_other_guard && + (entries_[it->first].propagate_up_at_count > 1); + } + + if (have_other_guard) { + for (StdIntMap::iterator it = parents->begin(); + it != parents->end(); ++it) + entries_[it->first].propagate_up_at_count -= 1; + + parents->clear(); // Forget the parents + } + } + } + + if (ExtraDebug) + PrintDebugInfo(&nodes); +} + +Prefilter* PrefilterTree::CanonicalNode(NodeMap* nodes, Prefilter* node) { + std::string node_string = NodeString(node); + std::map::iterator iter = nodes->find(node_string); + if (iter == nodes->end()) + return NULL; + return (*iter).second; +} + +std::string PrefilterTree::NodeString(Prefilter* node) const { + // Adding the operation disambiguates AND/OR/atom nodes. + std::string s = StringPrintf("%d", node->op()) + ":"; + if (node->op() == Prefilter::ATOM) { + s += node->atom(); + } else { + for (size_t i = 0; i < node->subs()->size(); i++) { + if (i > 0) + s += ','; + s += StringPrintf("%d", (*node->subs())[i]->unique_id()); + } + } + return s; +} + +bool PrefilterTree::KeepNode(Prefilter* node) const { + if (node == NULL) + return false; + + switch (node->op()) { + default: + LOG(DFATAL) << "Unexpected op in KeepNode: " << node->op(); + return false; + + case Prefilter::ALL: + case Prefilter::NONE: + return false; + + case Prefilter::ATOM: + return node->atom().size() >= static_cast(min_atom_len_); + + case Prefilter::AND: { + int j = 0; + std::vector* subs = node->subs(); + for (size_t i = 0; i < subs->size(); i++) + if (KeepNode((*subs)[i])) + (*subs)[j++] = (*subs)[i]; + else + delete (*subs)[i]; + + subs->resize(j); + return j > 0; + } + + case Prefilter::OR: + for (size_t i = 0; i < node->subs()->size(); i++) + if (!KeepNode((*node->subs())[i])) + return false; + return true; + } +} + +void PrefilterTree::AssignUniqueIds(NodeMap* nodes, + std::vector* atom_vec) { + atom_vec->clear(); + + // Build vector of all filter nodes, sorted topologically + // from top to bottom in v. + std::vector v; + + // Add the top level nodes of each regexp prefilter. + for (size_t i = 0; i < prefilter_vec_.size(); i++) { + Prefilter* f = prefilter_vec_[i]; + if (f == NULL) + unfiltered_.push_back(static_cast(i)); + + // We push NULL also on to v, so that we maintain the + // mapping of index==regexpid for level=0 prefilter nodes. + v.push_back(f); + } + + // Now add all the descendant nodes. + for (size_t i = 0; i < v.size(); i++) { + Prefilter* f = v[i]; + if (f == NULL) + continue; + if (f->op() == Prefilter::AND || f->op() == Prefilter::OR) { + const std::vector& subs = *f->subs(); + for (size_t j = 0; j < subs.size(); j++) + v.push_back(subs[j]); + } + } + + // Identify unique nodes. + int unique_id = 0; + for (int i = static_cast(v.size()) - 1; i >= 0; i--) { + Prefilter *node = v[i]; + if (node == NULL) + continue; + node->set_unique_id(-1); + Prefilter* canonical = CanonicalNode(nodes, node); + if (canonical == NULL) { + // Any further nodes that have the same node string + // will find this node as the canonical node. + nodes->emplace(NodeString(node), node); + if (node->op() == Prefilter::ATOM) { + atom_vec->push_back(node->atom()); + atom_index_to_id_.push_back(unique_id); + } + node->set_unique_id(unique_id++); + } else { + node->set_unique_id(canonical->unique_id()); + } + } + entries_.resize(nodes->size()); + + // Create parent StdIntMap for the entries. + for (int i = static_cast(v.size()) - 1; i >= 0; i--) { + Prefilter* prefilter = v[i]; + if (prefilter == NULL) + continue; + + if (CanonicalNode(nodes, prefilter) != prefilter) + continue; + + Entry* entry = &entries_[prefilter->unique_id()]; + entry->parents = new StdIntMap(); + } + + // Fill the entries. + for (int i = static_cast(v.size()) - 1; i >= 0; i--) { + Prefilter* prefilter = v[i]; + if (prefilter == NULL) + continue; + + if (CanonicalNode(nodes, prefilter) != prefilter) + continue; + + Entry* entry = &entries_[prefilter->unique_id()]; + + switch (prefilter->op()) { + default: + case Prefilter::ALL: + LOG(DFATAL) << "Unexpected op: " << prefilter->op(); + return; + + case Prefilter::ATOM: + entry->propagate_up_at_count = 1; + break; + + case Prefilter::OR: + case Prefilter::AND: { + std::set uniq_child; + for (size_t j = 0; j < prefilter->subs()->size(); j++) { + Prefilter* child = (*prefilter->subs())[j]; + Prefilter* canonical = CanonicalNode(nodes, child); + if (canonical == NULL) { + LOG(DFATAL) << "Null canonical node"; + return; + } + int child_id = canonical->unique_id(); + uniq_child.insert(child_id); + // To the child, we want to add to parent indices. + Entry* child_entry = &entries_[child_id]; + if (child_entry->parents->find(prefilter->unique_id()) == + child_entry->parents->end()) { + (*child_entry->parents)[prefilter->unique_id()] = 1; + } + } + entry->propagate_up_at_count = prefilter->op() == Prefilter::AND + ? static_cast(uniq_child.size()) + : 1; + + break; + } + } + } + + // For top level nodes, populate regexp id. + for (size_t i = 0; i < prefilter_vec_.size(); i++) { + if (prefilter_vec_[i] == NULL) + continue; + int id = CanonicalNode(nodes, prefilter_vec_[i])->unique_id(); + DCHECK_LE(0, id); + Entry* entry = &entries_[id]; + entry->regexps.push_back(static_cast(i)); + } +} + +// Functions for triggering during search. +void PrefilterTree::RegexpsGivenStrings( + const std::vector& matched_atoms, + std::vector* regexps) const { + regexps->clear(); + if (!compiled_) { + // Some legacy users of PrefilterTree call Compile() before + // adding any regexps and expect Compile() to have no effect. + // This kludge is a counterpart to that kludge. + if (prefilter_vec_.empty()) + return; + + LOG(ERROR) << "RegexpsGivenStrings called before Compile."; + for (size_t i = 0; i < prefilter_vec_.size(); i++) + regexps->push_back(static_cast(i)); + } else { + IntMap regexps_map(static_cast(prefilter_vec_.size())); + std::vector matched_atom_ids; + for (size_t j = 0; j < matched_atoms.size(); j++) + matched_atom_ids.push_back(atom_index_to_id_[matched_atoms[j]]); + PropagateMatch(matched_atom_ids, ®exps_map); + for (IntMap::iterator it = regexps_map.begin(); + it != regexps_map.end(); + ++it) + regexps->push_back(it->index()); + + regexps->insert(regexps->end(), unfiltered_.begin(), unfiltered_.end()); + } + std::sort(regexps->begin(), regexps->end()); +} + +void PrefilterTree::PropagateMatch(const std::vector& atom_ids, + IntMap* regexps) const { + IntMap count(static_cast(entries_.size())); + IntMap work(static_cast(entries_.size())); + for (size_t i = 0; i < atom_ids.size(); i++) + work.set(atom_ids[i], 1); + for (IntMap::iterator it = work.begin(); it != work.end(); ++it) { + const Entry& entry = entries_[it->index()]; + // Record regexps triggered. + for (size_t i = 0; i < entry.regexps.size(); i++) + regexps->set(entry.regexps[i], 1); + int c; + // Pass trigger up to parents. + for (StdIntMap::iterator it = entry.parents->begin(); + it != entry.parents->end(); + ++it) { + int j = it->first; + const Entry& parent = entries_[j]; + // Delay until all the children have succeeded. + if (parent.propagate_up_at_count > 1) { + if (count.has_index(j)) { + c = count.get_existing(j) + 1; + count.set_existing(j, c); + } else { + c = 1; + count.set_new(j, c); + } + if (c < parent.propagate_up_at_count) + continue; + } + // Trigger the parent. + work.set(j, 1); + } + } +} + +// Debugging help. +void PrefilterTree::PrintPrefilter(int regexpid) { + LOG(ERROR) << DebugNodeString(prefilter_vec_[regexpid]); +} + +void PrefilterTree::PrintDebugInfo(NodeMap* nodes) { + LOG(ERROR) << "#Unique Atoms: " << atom_index_to_id_.size(); + LOG(ERROR) << "#Unique Nodes: " << entries_.size(); + + for (size_t i = 0; i < entries_.size(); i++) { + StdIntMap* parents = entries_[i].parents; + const std::vector& regexps = entries_[i].regexps; + LOG(ERROR) << "EntryId: " << i + << " N: " << parents->size() << " R: " << regexps.size(); + for (StdIntMap::iterator it = parents->begin(); it != parents->end(); ++it) + LOG(ERROR) << it->first; + } + LOG(ERROR) << "Map:"; + for (std::map::const_iterator iter = nodes->begin(); + iter != nodes->end(); ++iter) + LOG(ERROR) << "NodeId: " << (*iter).second->unique_id() + << " Str: " << (*iter).first; +} + +std::string PrefilterTree::DebugNodeString(Prefilter* node) const { + std::string node_string = ""; + if (node->op() == Prefilter::ATOM) { + DCHECK(!node->atom().empty()); + node_string += node->atom(); + } else { + // Adding the operation disambiguates AND and OR nodes. + node_string += node->op() == Prefilter::AND ? "AND" : "OR"; + node_string += "("; + for (size_t i = 0; i < node->subs()->size(); i++) { + if (i > 0) + node_string += ','; + node_string += StringPrintf("%d", (*node->subs())[i]->unique_id()); + node_string += ":"; + node_string += DebugNodeString((*node->subs())[i]); + } + node_string += ")"; + } + return node_string; +} + +} // namespace re2 diff --git a/extern/re2/re2/prefilter_tree.h b/extern/re2/re2/prefilter_tree.h new file mode 100644 index 0000000000..b2e2d749b3 --- /dev/null +++ b/extern/re2/re2/prefilter_tree.h @@ -0,0 +1,139 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_PREFILTER_TREE_H_ +#define RE2_PREFILTER_TREE_H_ + +// The PrefilterTree class is used to form an AND-OR tree of strings +// that would trigger each regexp. The 'prefilter' of each regexp is +// added to PrefilterTree, and then PrefilterTree is used to find all +// the unique strings across the prefilters. During search, by using +// matches from a string matching engine, PrefilterTree deduces the +// set of regexps that are to be triggered. The 'string matching +// engine' itself is outside of this class, and the caller can use any +// favorite engine. PrefilterTree provides a set of strings (called +// atoms) that the user of this class should use to do the string +// matching. + +#include +#include +#include + +#include "util/util.h" +#include "util/sparse_array.h" +#include "re2/prefilter.h" + +namespace re2 { + +class PrefilterTree { + public: + PrefilterTree(); + explicit PrefilterTree(int min_atom_len); + ~PrefilterTree(); + + // Adds the prefilter for the next regexp. Note that we assume that + // Add called sequentially for all regexps. All Add calls + // must precede Compile. + void Add(Prefilter* prefilter); + + // The Compile returns a vector of string in atom_vec. + // Call this after all the prefilters are added through Add. + // No calls to Add after Compile are allowed. + // The caller should use the returned set of strings to do string matching. + // Each time a string matches, the corresponding index then has to be + // and passed to RegexpsGivenStrings below. + void Compile(std::vector* atom_vec); + + // Given the indices of the atoms that matched, returns the indexes + // of regexps that should be searched. The matched_atoms should + // contain all the ids of string atoms that were found to match the + // content. The caller can use any string match engine to perform + // this function. This function is thread safe. + void RegexpsGivenStrings(const std::vector& matched_atoms, + std::vector* regexps) const; + + // Print debug prefilter. Also prints unique ids associated with + // nodes of the prefilter of the regexp. + void PrintPrefilter(int regexpid); + + private: + typedef SparseArray IntMap; + typedef std::map StdIntMap; + typedef std::map NodeMap; + + // Each unique node has a corresponding Entry that helps in + // passing the matching trigger information along the tree. + struct Entry { + public: + // How many children should match before this node triggers the + // parent. For an atom and an OR node, this is 1 and for an AND + // node, it is the number of unique children. + int propagate_up_at_count; + + // When this node is ready to trigger the parent, what are the indices + // of the parent nodes to trigger. The reason there may be more than + // one is because of sharing. For example (abc | def) and (xyz | def) + // are two different nodes, but they share the atom 'def'. So when + // 'def' matches, it triggers two parents, corresponding to the two + // different OR nodes. + StdIntMap* parents; + + // When this node is ready to trigger the parent, what are the + // regexps that are triggered. + std::vector regexps; + }; + + // Returns true if the prefilter node should be kept. + bool KeepNode(Prefilter* node) const; + + // This function assigns unique ids to various parts of the + // prefilter, by looking at if these nodes are already in the + // PrefilterTree. + void AssignUniqueIds(NodeMap* nodes, std::vector* atom_vec); + + // Given the matching atoms, find the regexps to be triggered. + void PropagateMatch(const std::vector& atom_ids, + IntMap* regexps) const; + + // Returns the prefilter node that has the same NodeString as this + // node. For the canonical node, returns node. + Prefilter* CanonicalNode(NodeMap* nodes, Prefilter* node); + + // A string that uniquely identifies the node. Assumes that the + // children of node has already been assigned unique ids. + std::string NodeString(Prefilter* node) const; + + // Recursively constructs a readable prefilter string. + std::string DebugNodeString(Prefilter* node) const; + + // Used for debugging. + void PrintDebugInfo(NodeMap* nodes); + + // These are all the nodes formed by Compile. Essentially, there is + // one node for each unique atom and each unique AND/OR node. + std::vector entries_; + + // indices of regexps that always pass through the filter (since we + // found no required literals in these regexps). + std::vector unfiltered_; + + // vector of Prefilter for all regexps. + std::vector prefilter_vec_; + + // Atom index in returned strings to entry id mapping. + std::vector atom_index_to_id_; + + // Has the prefilter tree been compiled. + bool compiled_; + + // Strings less than this length are not stored as atoms. + const int min_atom_len_; + + PrefilterTree(const PrefilterTree&) = delete; + PrefilterTree& operator=(const PrefilterTree&) = delete; +}; + +} // namespace + +#endif // RE2_PREFILTER_TREE_H_ diff --git a/extern/re2/re2/prog.cc b/extern/re2/re2/prog.cc new file mode 100644 index 0000000000..5155943cb1 --- /dev/null +++ b/extern/re2/re2/prog.cc @@ -0,0 +1,921 @@ +// Copyright 2007 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Compiled regular expression representation. +// Tested by compile_test.cc + +#include "re2/prog.h" + +#include +#include +#include +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "util/strutil.h" +#include "re2/bitmap256.h" +#include "re2/stringpiece.h" + +namespace re2 { + +// Constructors per Inst opcode + +void Prog::Inst::InitAlt(uint32_t out, uint32_t out1) { + DCHECK_EQ(out_opcode_, 0); + set_out_opcode(out, kInstAlt); + out1_ = out1; +} + +void Prog::Inst::InitByteRange(int lo, int hi, int foldcase, uint32_t out) { + DCHECK_EQ(out_opcode_, 0); + set_out_opcode(out, kInstByteRange); + lo_ = lo & 0xFF; + hi_ = hi & 0xFF; + hint_foldcase_ = foldcase&1; +} + +void Prog::Inst::InitCapture(int cap, uint32_t out) { + DCHECK_EQ(out_opcode_, 0); + set_out_opcode(out, kInstCapture); + cap_ = cap; +} + +void Prog::Inst::InitEmptyWidth(EmptyOp empty, uint32_t out) { + DCHECK_EQ(out_opcode_, 0); + set_out_opcode(out, kInstEmptyWidth); + empty_ = empty; +} + +void Prog::Inst::InitMatch(int32_t id) { + DCHECK_EQ(out_opcode_, 0); + set_opcode(kInstMatch); + match_id_ = id; +} + +void Prog::Inst::InitNop(uint32_t out) { + DCHECK_EQ(out_opcode_, 0); + set_opcode(kInstNop); +} + +void Prog::Inst::InitFail() { + DCHECK_EQ(out_opcode_, 0); + set_opcode(kInstFail); +} + +std::string Prog::Inst::Dump() { + switch (opcode()) { + default: + return StringPrintf("opcode %d", static_cast(opcode())); + + case kInstAlt: + return StringPrintf("alt -> %d | %d", out(), out1_); + + case kInstAltMatch: + return StringPrintf("altmatch -> %d | %d", out(), out1_); + + case kInstByteRange: + return StringPrintf("byte%s [%02x-%02x] %d -> %d", + foldcase() ? "/i" : "", + lo_, hi_, hint(), out()); + + case kInstCapture: + return StringPrintf("capture %d -> %d", cap_, out()); + + case kInstEmptyWidth: + return StringPrintf("emptywidth %#x -> %d", + static_cast(empty_), out()); + + case kInstMatch: + return StringPrintf("match! %d", match_id()); + + case kInstNop: + return StringPrintf("nop -> %d", out()); + + case kInstFail: + return StringPrintf("fail"); + } +} + +Prog::Prog() + : anchor_start_(false), + anchor_end_(false), + reversed_(false), + did_flatten_(false), + did_onepass_(false), + start_(0), + start_unanchored_(0), + size_(0), + bytemap_range_(0), + first_byte_(-1), + flags_(0), + list_count_(0), + dfa_mem_(0), + dfa_first_(NULL), + dfa_longest_(NULL) { +} + +Prog::~Prog() { + DeleteDFA(dfa_longest_); + DeleteDFA(dfa_first_); +} + +typedef SparseSet Workq; + +static inline void AddToQueue(Workq* q, int id) { + if (id != 0) + q->insert(id); +} + +static std::string ProgToString(Prog* prog, Workq* q) { + std::string s; + for (Workq::iterator i = q->begin(); i != q->end(); ++i) { + int id = *i; + Prog::Inst* ip = prog->inst(id); + s += StringPrintf("%d. %s\n", id, ip->Dump().c_str()); + AddToQueue(q, ip->out()); + if (ip->opcode() == kInstAlt || ip->opcode() == kInstAltMatch) + AddToQueue(q, ip->out1()); + } + return s; +} + +static std::string FlattenedProgToString(Prog* prog, int start) { + std::string s; + for (int id = start; id < prog->size(); id++) { + Prog::Inst* ip = prog->inst(id); + if (ip->last()) + s += StringPrintf("%d. %s\n", id, ip->Dump().c_str()); + else + s += StringPrintf("%d+ %s\n", id, ip->Dump().c_str()); + } + return s; +} + +std::string Prog::Dump() { + if (did_flatten_) + return FlattenedProgToString(this, start_); + + Workq q(size_); + AddToQueue(&q, start_); + return ProgToString(this, &q); +} + +std::string Prog::DumpUnanchored() { + if (did_flatten_) + return FlattenedProgToString(this, start_unanchored_); + + Workq q(size_); + AddToQueue(&q, start_unanchored_); + return ProgToString(this, &q); +} + +std::string Prog::DumpByteMap() { + std::string map; + for (int c = 0; c < 256; c++) { + int b = bytemap_[c]; + int lo = c; + while (c < 256-1 && bytemap_[c+1] == b) + c++; + int hi = c; + map += StringPrintf("[%02x-%02x] -> %d\n", lo, hi, b); + } + return map; +} + +int Prog::first_byte() { + std::call_once(first_byte_once_, [](Prog* prog) { + prog->first_byte_ = prog->ComputeFirstByte(); + }, this); + return first_byte_; +} + +static bool IsMatch(Prog*, Prog::Inst*); + +// Peep-hole optimizer. +void Prog::Optimize() { + Workq q(size_); + + // Eliminate nops. Most are taken out during compilation + // but a few are hard to avoid. + q.clear(); + AddToQueue(&q, start_); + for (Workq::iterator i = q.begin(); i != q.end(); ++i) { + int id = *i; + + Inst* ip = inst(id); + int j = ip->out(); + Inst* jp; + while (j != 0 && (jp=inst(j))->opcode() == kInstNop) { + j = jp->out(); + } + ip->set_out(j); + AddToQueue(&q, ip->out()); + + if (ip->opcode() == kInstAlt) { + j = ip->out1(); + while (j != 0 && (jp=inst(j))->opcode() == kInstNop) { + j = jp->out(); + } + ip->out1_ = j; + AddToQueue(&q, ip->out1()); + } + } + + // Insert kInstAltMatch instructions + // Look for + // ip: Alt -> j | k + // j: ByteRange [00-FF] -> ip + // k: Match + // or the reverse (the above is the greedy one). + // Rewrite Alt to AltMatch. + q.clear(); + AddToQueue(&q, start_); + for (Workq::iterator i = q.begin(); i != q.end(); ++i) { + int id = *i; + Inst* ip = inst(id); + AddToQueue(&q, ip->out()); + if (ip->opcode() == kInstAlt) + AddToQueue(&q, ip->out1()); + + if (ip->opcode() == kInstAlt) { + Inst* j = inst(ip->out()); + Inst* k = inst(ip->out1()); + if (j->opcode() == kInstByteRange && j->out() == id && + j->lo() == 0x00 && j->hi() == 0xFF && + IsMatch(this, k)) { + ip->set_opcode(kInstAltMatch); + continue; + } + if (IsMatch(this, j) && + k->opcode() == kInstByteRange && k->out() == id && + k->lo() == 0x00 && k->hi() == 0xFF) { + ip->set_opcode(kInstAltMatch); + } + } + } +} + +// Is ip a guaranteed match at end of text, perhaps after some capturing? +static bool IsMatch(Prog* prog, Prog::Inst* ip) { + for (;;) { + switch (ip->opcode()) { + default: + LOG(DFATAL) << "Unexpected opcode in IsMatch: " << ip->opcode(); + return false; + + case kInstAlt: + case kInstAltMatch: + case kInstByteRange: + case kInstFail: + case kInstEmptyWidth: + return false; + + case kInstCapture: + case kInstNop: + ip = prog->inst(ip->out()); + break; + + case kInstMatch: + return true; + } + } +} + +uint32_t Prog::EmptyFlags(const StringPiece& text, const char* p) { + int flags = 0; + + // ^ and \A + if (p == text.begin()) + flags |= kEmptyBeginText | kEmptyBeginLine; + else if (p[-1] == '\n') + flags |= kEmptyBeginLine; + + // $ and \z + if (p == text.end()) + flags |= kEmptyEndText | kEmptyEndLine; + else if (p < text.end() && p[0] == '\n') + flags |= kEmptyEndLine; + + // \b and \B + if (p == text.begin() && p == text.end()) { + // no word boundary here + } else if (p == text.begin()) { + if (IsWordChar(p[0])) + flags |= kEmptyWordBoundary; + } else if (p == text.end()) { + if (IsWordChar(p[-1])) + flags |= kEmptyWordBoundary; + } else { + if (IsWordChar(p[-1]) != IsWordChar(p[0])) + flags |= kEmptyWordBoundary; + } + if (!(flags & kEmptyWordBoundary)) + flags |= kEmptyNonWordBoundary; + + return flags; +} + +// ByteMapBuilder implements a coloring algorithm. +// +// The first phase is a series of "mark and merge" batches: we mark one or more +// [lo-hi] ranges, then merge them into our internal state. Batching is not for +// performance; rather, it means that the ranges are treated indistinguishably. +// +// Internally, the ranges are represented using a bitmap that stores the splits +// and a vector that stores the colors; both of them are indexed by the ranges' +// last bytes. Thus, in order to merge a [lo-hi] range, we split at lo-1 and at +// hi (if not already split), then recolor each range in between. The color map +// (i.e. from the old color to the new color) is maintained for the lifetime of +// the batch and so underpins this somewhat obscure approach to set operations. +// +// The second phase builds the bytemap from our internal state: we recolor each +// range, then store the new color (which is now the byte class) in each of the +// corresponding array elements. Finally, we output the number of byte classes. +class ByteMapBuilder { + public: + ByteMapBuilder() { + // Initial state: the [0-255] range has color 256. + // This will avoid problems during the second phase, + // in which we assign byte classes numbered from 0. + splits_.Set(255); + colors_[255] = 256; + nextcolor_ = 257; + } + + void Mark(int lo, int hi); + void Merge(); + void Build(uint8_t* bytemap, int* bytemap_range); + + private: + int Recolor(int oldcolor); + + Bitmap256 splits_; + int colors_[256]; + int nextcolor_; + std::vector> colormap_; + std::vector> ranges_; + + ByteMapBuilder(const ByteMapBuilder&) = delete; + ByteMapBuilder& operator=(const ByteMapBuilder&) = delete; +}; + +void ByteMapBuilder::Mark(int lo, int hi) { + DCHECK_GE(lo, 0); + DCHECK_GE(hi, 0); + DCHECK_LE(lo, 255); + DCHECK_LE(hi, 255); + DCHECK_LE(lo, hi); + + // Ignore any [0-255] ranges. They cause us to recolor every range, which + // has no effect on the eventual result and is therefore a waste of time. + if (lo == 0 && hi == 255) + return; + + ranges_.emplace_back(lo, hi); +} + +void ByteMapBuilder::Merge() { + for (std::vector>::const_iterator it = ranges_.begin(); + it != ranges_.end(); + ++it) { + int lo = it->first-1; + int hi = it->second; + + if (0 <= lo && !splits_.Test(lo)) { + splits_.Set(lo); + int next = splits_.FindNextSetBit(lo+1); + colors_[lo] = colors_[next]; + } + if (!splits_.Test(hi)) { + splits_.Set(hi); + int next = splits_.FindNextSetBit(hi+1); + colors_[hi] = colors_[next]; + } + + int c = lo+1; + while (c < 256) { + int next = splits_.FindNextSetBit(c); + colors_[next] = Recolor(colors_[next]); + if (next == hi) + break; + c = next+1; + } + } + colormap_.clear(); + ranges_.clear(); +} + +void ByteMapBuilder::Build(uint8_t* bytemap, int* bytemap_range) { + // Assign byte classes numbered from 0. + nextcolor_ = 0; + + int c = 0; + while (c < 256) { + int next = splits_.FindNextSetBit(c); + uint8_t b = static_cast(Recolor(colors_[next])); + while (c <= next) { + bytemap[c] = b; + c++; + } + } + + *bytemap_range = nextcolor_; +} + +int ByteMapBuilder::Recolor(int oldcolor) { + // Yes, this is a linear search. There can be at most 256 + // colors and there will typically be far fewer than that. + // Also, we need to consider keys *and* values in order to + // avoid recoloring a given range more than once per batch. + std::vector>::const_iterator it = + std::find_if(colormap_.begin(), colormap_.end(), + [=](const std::pair& kv) -> bool { + return kv.first == oldcolor || kv.second == oldcolor; + }); + if (it != colormap_.end()) + return it->second; + int newcolor = nextcolor_; + nextcolor_++; + colormap_.emplace_back(oldcolor, newcolor); + return newcolor; +} + +void Prog::ComputeByteMap() { + // Fill in bytemap with byte classes for the program. + // Ranges of bytes that are treated indistinguishably + // will be mapped to a single byte class. + ByteMapBuilder builder; + + // Don't repeat the work for ^ and $. + bool marked_line_boundaries = false; + // Don't repeat the work for \b and \B. + bool marked_word_boundaries = false; + + for (int id = 0; id < size(); id++) { + Inst* ip = inst(id); + if (ip->opcode() == kInstByteRange) { + int lo = ip->lo(); + int hi = ip->hi(); + builder.Mark(lo, hi); + if (ip->foldcase() && lo <= 'z' && hi >= 'a') { + int foldlo = lo; + int foldhi = hi; + if (foldlo < 'a') + foldlo = 'a'; + if (foldhi > 'z') + foldhi = 'z'; + if (foldlo <= foldhi) { + foldlo += 'A' - 'a'; + foldhi += 'A' - 'a'; + builder.Mark(foldlo, foldhi); + } + } + // If this Inst is not the last Inst in its list AND the next Inst is + // also a ByteRange AND the Insts have the same out, defer the merge. + if (!ip->last() && + inst(id+1)->opcode() == kInstByteRange && + ip->out() == inst(id+1)->out()) + continue; + builder.Merge(); + } else if (ip->opcode() == kInstEmptyWidth) { + if (ip->empty() & (kEmptyBeginLine|kEmptyEndLine) && + !marked_line_boundaries) { + builder.Mark('\n', '\n'); + builder.Merge(); + marked_line_boundaries = true; + } + if (ip->empty() & (kEmptyWordBoundary|kEmptyNonWordBoundary) && + !marked_word_boundaries) { + // We require two batches here: the first for ranges that are word + // characters, the second for ranges that are not word characters. + for (bool isword : {true, false}) { + int j; + for (int i = 0; i < 256; i = j) { + for (j = i + 1; j < 256 && + Prog::IsWordChar(static_cast(i)) == + Prog::IsWordChar(static_cast(j)); + j++) + ; + if (Prog::IsWordChar(static_cast(i)) == isword) + builder.Mark(i, j - 1); + } + builder.Merge(); + } + marked_word_boundaries = true; + } + } + } + + builder.Build(bytemap_, &bytemap_range_); + + if (0) { // For debugging, use trivial bytemap. + LOG(ERROR) << "Using trivial bytemap."; + for (int i = 0; i < 256; i++) + bytemap_[i] = static_cast(i); + bytemap_range_ = 256; + } +} + +// Prog::Flatten() implements a graph rewriting algorithm. +// +// The overall process is similar to epsilon removal, but retains some epsilon +// transitions: those from Capture and EmptyWidth instructions; and those from +// nullable subexpressions. (The latter avoids quadratic blowup in transitions +// in the worst case.) It might be best thought of as Alt instruction elision. +// +// In conceptual terms, it divides the Prog into "trees" of instructions, then +// traverses the "trees" in order to produce "lists" of instructions. A "tree" +// is one or more instructions that grow from one "root" instruction to one or +// more "leaf" instructions; if a "tree" has exactly one instruction, then the +// "root" is also the "leaf". In most cases, a "root" is the successor of some +// "leaf" (i.e. the "leaf" instruction's out() returns the "root" instruction) +// and is considered a "successor root". A "leaf" can be a ByteRange, Capture, +// EmptyWidth or Match instruction. However, this is insufficient for handling +// nested nullable subexpressions correctly, so in some cases, a "root" is the +// dominator of the instructions reachable from some "successor root" (i.e. it +// has an unreachable predecessor) and is considered a "dominator root". Since +// only Alt instructions can be "dominator roots" (other instructions would be +// "leaves"), only Alt instructions are required to be marked as predecessors. +// +// Dividing the Prog into "trees" comprises two passes: marking the "successor +// roots" and the predecessors; and marking the "dominator roots". Sorting the +// "successor roots" by their bytecode offsets enables iteration in order from +// greatest to least during the second pass; by working backwards in this case +// and flooding the graph no further than "leaves" and already marked "roots", +// it becomes possible to mark "dominator roots" without doing excessive work. +// +// Traversing the "trees" is just iterating over the "roots" in order of their +// marking and flooding the graph no further than "leaves" and "roots". When a +// "leaf" is reached, the instruction is copied with its successor remapped to +// its "root" number. When a "root" is reached, a Nop instruction is generated +// with its successor remapped similarly. As each "list" is produced, its last +// instruction is marked as such. After all of the "lists" have been produced, +// a pass over their instructions remaps their successors to bytecode offsets. +void Prog::Flatten() { + if (did_flatten_) + return; + did_flatten_ = true; + + // Scratch structures. It's important that these are reused by functions + // that we call in loops because they would thrash the heap otherwise. + SparseSet reachable(size()); + std::vector stk; + stk.reserve(size()); + + // First pass: Marks "successor roots" and predecessors. + // Builds the mapping from inst-ids to root-ids. + SparseArray rootmap(size()); + SparseArray predmap(size()); + std::vector> predvec; + MarkSuccessors(&rootmap, &predmap, &predvec, &reachable, &stk); + + // Second pass: Marks "dominator roots". + SparseArray sorted(rootmap); + std::sort(sorted.begin(), sorted.end(), sorted.less); + for (SparseArray::const_iterator i = sorted.end() - 1; + i != sorted.begin(); + --i) { + if (i->index() != start_unanchored() && i->index() != start()) + MarkDominator(i->index(), &rootmap, &predmap, &predvec, &reachable, &stk); + } + + // Third pass: Emits "lists". Remaps outs to root-ids. + // Builds the mapping from root-ids to flat-ids. + std::vector flatmap(rootmap.size()); + std::vector flat; + flat.reserve(size()); + for (SparseArray::const_iterator i = rootmap.begin(); + i != rootmap.end(); + ++i) { + flatmap[i->value()] = static_cast(flat.size()); + EmitList(i->index(), &rootmap, &flat, &reachable, &stk); + flat.back().set_last(); + // We have the bounds of the "list", so this is the + // most convenient point at which to compute hints. + ComputeHints(&flat, flatmap[i->value()], static_cast(flat.size())); + } + + list_count_ = static_cast(flatmap.size()); + for (int i = 0; i < kNumInst; i++) + inst_count_[i] = 0; + + // Fourth pass: Remaps outs to flat-ids. + // Counts instructions by opcode. + for (int id = 0; id < static_cast(flat.size()); id++) { + Inst* ip = &flat[id]; + if (ip->opcode() != kInstAltMatch) // handled in EmitList() + ip->set_out(flatmap[ip->out()]); + inst_count_[ip->opcode()]++; + } + + int total = 0; + for (int i = 0; i < kNumInst; i++) + total += inst_count_[i]; + DCHECK_EQ(total, static_cast(flat.size())); + + // Remap start_unanchored and start. + if (start_unanchored() == 0) { + DCHECK_EQ(start(), 0); + } else if (start_unanchored() == start()) { + set_start_unanchored(flatmap[1]); + set_start(flatmap[1]); + } else { + set_start_unanchored(flatmap[1]); + set_start(flatmap[2]); + } + + // Finally, replace the old instructions with the new instructions. + size_ = static_cast(flat.size()); + inst_ = PODArray(size_); + memmove(inst_.data(), flat.data(), size_*sizeof inst_[0]); + + // Populate the list heads for BitState. + // 512 instructions limits the memory footprint to 1KiB. + if (size_ <= 512) { + list_heads_ = PODArray(size_); + // 0xFF makes it more obvious if we try to look up a non-head. + memset(list_heads_.data(), 0xFF, size_*sizeof list_heads_[0]); + for (int i = 0; i < list_count_; ++i) + list_heads_[flatmap[i]] = i; + } +} + +void Prog::MarkSuccessors(SparseArray* rootmap, + SparseArray* predmap, + std::vector>* predvec, + SparseSet* reachable, std::vector* stk) { + // Mark the kInstFail instruction. + rootmap->set_new(0, rootmap->size()); + + // Mark the start_unanchored and start instructions. + if (!rootmap->has_index(start_unanchored())) + rootmap->set_new(start_unanchored(), rootmap->size()); + if (!rootmap->has_index(start())) + rootmap->set_new(start(), rootmap->size()); + + reachable->clear(); + stk->clear(); + stk->push_back(start_unanchored()); + while (!stk->empty()) { + int id = stk->back(); + stk->pop_back(); + Loop: + if (reachable->contains(id)) + continue; + reachable->insert_new(id); + + Inst* ip = inst(id); + switch (ip->opcode()) { + default: + LOG(DFATAL) << "unhandled opcode: " << ip->opcode(); + break; + + case kInstAltMatch: + case kInstAlt: + // Mark this instruction as a predecessor of each out. + for (int out : {ip->out(), ip->out1()}) { + if (!predmap->has_index(out)) { + predmap->set_new(out, static_cast(predvec->size())); + predvec->emplace_back(); + } + (*predvec)[predmap->get_existing(out)].emplace_back(id); + } + stk->push_back(ip->out1()); + id = ip->out(); + goto Loop; + + case kInstByteRange: + case kInstCapture: + case kInstEmptyWidth: + // Mark the out of this instruction as a "root". + if (!rootmap->has_index(ip->out())) + rootmap->set_new(ip->out(), rootmap->size()); + id = ip->out(); + goto Loop; + + case kInstNop: + id = ip->out(); + goto Loop; + + case kInstMatch: + case kInstFail: + break; + } + } +} + +void Prog::MarkDominator(int root, SparseArray* rootmap, + SparseArray* predmap, + std::vector>* predvec, + SparseSet* reachable, std::vector* stk) { + reachable->clear(); + stk->clear(); + stk->push_back(root); + while (!stk->empty()) { + int id = stk->back(); + stk->pop_back(); + Loop: + if (reachable->contains(id)) + continue; + reachable->insert_new(id); + + if (id != root && rootmap->has_index(id)) { + // We reached another "tree" via epsilon transition. + continue; + } + + Inst* ip = inst(id); + switch (ip->opcode()) { + default: + LOG(DFATAL) << "unhandled opcode: " << ip->opcode(); + break; + + case kInstAltMatch: + case kInstAlt: + stk->push_back(ip->out1()); + id = ip->out(); + goto Loop; + + case kInstByteRange: + case kInstCapture: + case kInstEmptyWidth: + break; + + case kInstNop: + id = ip->out(); + goto Loop; + + case kInstMatch: + case kInstFail: + break; + } + } + + for (SparseSet::const_iterator i = reachable->begin(); + i != reachable->end(); + ++i) { + int id = *i; + if (predmap->has_index(id)) { + for (int pred : (*predvec)[predmap->get_existing(id)]) { + if (!reachable->contains(pred)) { + // id has a predecessor that cannot be reached from root! + // Therefore, id must be a "root" too - mark it as such. + if (!rootmap->has_index(id)) + rootmap->set_new(id, rootmap->size()); + } + } + } + } +} + +void Prog::EmitList(int root, SparseArray* rootmap, + std::vector* flat, + SparseSet* reachable, std::vector* stk) { + reachable->clear(); + stk->clear(); + stk->push_back(root); + while (!stk->empty()) { + int id = stk->back(); + stk->pop_back(); + Loop: + if (reachable->contains(id)) + continue; + reachable->insert_new(id); + + if (id != root && rootmap->has_index(id)) { + // We reached another "tree" via epsilon transition. Emit a kInstNop + // instruction so that the Prog does not become quadratically larger. + flat->emplace_back(); + flat->back().set_opcode(kInstNop); + flat->back().set_out(rootmap->get_existing(id)); + continue; + } + + Inst* ip = inst(id); + switch (ip->opcode()) { + default: + LOG(DFATAL) << "unhandled opcode: " << ip->opcode(); + break; + + case kInstAltMatch: + flat->emplace_back(); + flat->back().set_opcode(kInstAltMatch); + flat->back().set_out(static_cast(flat->size())); + flat->back().out1_ = static_cast(flat->size())+1; + FALLTHROUGH_INTENDED; + + case kInstAlt: + stk->push_back(ip->out1()); + id = ip->out(); + goto Loop; + + case kInstByteRange: + case kInstCapture: + case kInstEmptyWidth: + flat->emplace_back(); + memmove(&flat->back(), ip, sizeof *ip); + flat->back().set_out(rootmap->get_existing(ip->out())); + break; + + case kInstNop: + id = ip->out(); + goto Loop; + + case kInstMatch: + case kInstFail: + flat->emplace_back(); + memmove(&flat->back(), ip, sizeof *ip); + break; + } + } +} + +// For each ByteRange instruction in [begin, end), computes a hint to execution +// engines: the delta to the next instruction (in flat) worth exploring iff the +// current instruction matched. +// +// Implements a coloring algorithm related to ByteMapBuilder, but in this case, +// colors are instructions and recoloring ranges precisely identifies conflicts +// between instructions. Iterating backwards over [begin, end) is guaranteed to +// identify the nearest conflict (if any) with only linear complexity. +void Prog::ComputeHints(std::vector* flat, int begin, int end) { + Bitmap256 splits; + int colors[256]; + + bool dirty = false; + for (int id = end; id >= begin; --id) { + if (id == end || + (*flat)[id].opcode() != kInstByteRange) { + if (dirty) { + dirty = false; + splits.Clear(); + } + splits.Set(255); + colors[255] = id; + // At this point, the [0-255] range is colored with id. + // Thus, hints cannot point beyond id; and if id == end, + // hints that would have pointed to id will be 0 instead. + continue; + } + dirty = true; + + // We recolor the [lo-hi] range with id. Note that first ratchets backwards + // from end to the nearest conflict (if any) during recoloring. + int first = end; + auto Recolor = [&](int lo, int hi) { + // Like ByteMapBuilder, we split at lo-1 and at hi. + --lo; + + if (0 <= lo && !splits.Test(lo)) { + splits.Set(lo); + int next = splits.FindNextSetBit(lo+1); + colors[lo] = colors[next]; + } + if (!splits.Test(hi)) { + splits.Set(hi); + int next = splits.FindNextSetBit(hi+1); + colors[hi] = colors[next]; + } + + int c = lo+1; + while (c < 256) { + int next = splits.FindNextSetBit(c); + // Ratchet backwards... + first = std::min(first, colors[next]); + // Recolor with id - because it's the new nearest conflict! + colors[next] = id; + if (next == hi) + break; + c = next+1; + } + }; + + Inst* ip = &(*flat)[id]; + int lo = ip->lo(); + int hi = ip->hi(); + Recolor(lo, hi); + if (ip->foldcase() && lo <= 'z' && hi >= 'a') { + int foldlo = lo; + int foldhi = hi; + if (foldlo < 'a') + foldlo = 'a'; + if (foldhi > 'z') + foldhi = 'z'; + if (foldlo <= foldhi) { + foldlo += 'A' - 'a'; + foldhi += 'A' - 'a'; + Recolor(foldlo, foldhi); + } + } + + if (first != end) { + uint16_t hint = static_cast(std::min(first - id, 32767)); + ip->hint_foldcase_ |= hint<<1; + } + } +} + +} // namespace re2 diff --git a/extern/re2/re2/prog.h b/extern/re2/re2/prog.h new file mode 100644 index 0000000000..bacc411797 --- /dev/null +++ b/extern/re2/re2/prog.h @@ -0,0 +1,432 @@ +// Copyright 2007 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_PROG_H_ +#define RE2_PROG_H_ + +// Compiled representation of regular expressions. +// See regexp.h for the Regexp class, which represents a regular +// expression symbolically. + +#include +#include +#include +#include +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "util/pod_array.h" +#include "util/sparse_array.h" +#include "util/sparse_set.h" +#include "re2/re2.h" + +namespace re2 { + +// Opcodes for Inst +enum InstOp { + kInstAlt = 0, // choose between out_ and out1_ + kInstAltMatch, // Alt: out_ is [00-FF] and back, out1_ is match; or vice versa. + kInstByteRange, // next (possible case-folded) byte must be in [lo_, hi_] + kInstCapture, // capturing parenthesis number cap_ + kInstEmptyWidth, // empty-width special (^ $ ...); bit(s) set in empty_ + kInstMatch, // found a match! + kInstNop, // no-op; occasionally unavoidable + kInstFail, // never match; occasionally unavoidable + kNumInst, +}; + +// Bit flags for empty-width specials +enum EmptyOp { + kEmptyBeginLine = 1<<0, // ^ - beginning of line + kEmptyEndLine = 1<<1, // $ - end of line + kEmptyBeginText = 1<<2, // \A - beginning of text + kEmptyEndText = 1<<3, // \z - end of text + kEmptyWordBoundary = 1<<4, // \b - word boundary + kEmptyNonWordBoundary = 1<<5, // \B - not \b + kEmptyAllFlags = (1<<6)-1, +}; + +class DFA; +class Regexp; + +// Compiled form of regexp program. +class Prog { + public: + Prog(); + ~Prog(); + + // Single instruction in regexp program. + class Inst { + public: + // See the assertion below for why this is so. + Inst() = default; + + // Copyable. + Inst(const Inst&) = default; + Inst& operator=(const Inst&) = default; + + // Constructors per opcode + void InitAlt(uint32_t out, uint32_t out1); + void InitByteRange(int lo, int hi, int foldcase, uint32_t out); + void InitCapture(int cap, uint32_t out); + void InitEmptyWidth(EmptyOp empty, uint32_t out); + void InitMatch(int id); + void InitNop(uint32_t out); + void InitFail(); + + // Getters + int id(Prog* p) { return static_cast(this - p->inst_.data()); } + InstOp opcode() { return static_cast(out_opcode_&7); } + int last() { return (out_opcode_>>3)&1; } + int out() { return out_opcode_>>4; } + int out1() { DCHECK(opcode() == kInstAlt || opcode() == kInstAltMatch); return out1_; } + int cap() { DCHECK_EQ(opcode(), kInstCapture); return cap_; } + int lo() { DCHECK_EQ(opcode(), kInstByteRange); return lo_; } + int hi() { DCHECK_EQ(opcode(), kInstByteRange); return hi_; } + int foldcase() { DCHECK_EQ(opcode(), kInstByteRange); return hint_foldcase_&1; } + int hint() { DCHECK_EQ(opcode(), kInstByteRange); return hint_foldcase_>>1; } + int match_id() { DCHECK_EQ(opcode(), kInstMatch); return match_id_; } + EmptyOp empty() { DCHECK_EQ(opcode(), kInstEmptyWidth); return empty_; } + + bool greedy(Prog* p) { + DCHECK_EQ(opcode(), kInstAltMatch); + return p->inst(out())->opcode() == kInstByteRange || + (p->inst(out())->opcode() == kInstNop && + p->inst(p->inst(out())->out())->opcode() == kInstByteRange); + } + + // Does this inst (an kInstByteRange) match c? + inline bool Matches(int c) { + DCHECK_EQ(opcode(), kInstByteRange); + if (foldcase() && 'A' <= c && c <= 'Z') + c += 'a' - 'A'; + return lo_ <= c && c <= hi_; + } + + // Returns string representation for debugging. + std::string Dump(); + + // Maximum instruction id. + // (Must fit in out_opcode_. PatchList/last steal another bit.) + static const int kMaxInst = (1<<28) - 1; + + private: + void set_opcode(InstOp opcode) { + out_opcode_ = (out()<<4) | (last()<<3) | opcode; + } + + void set_last() { + out_opcode_ = (out()<<4) | (1<<3) | opcode(); + } + + void set_out(int out) { + out_opcode_ = (out<<4) | (last()<<3) | opcode(); + } + + void set_out_opcode(int out, InstOp opcode) { + out_opcode_ = (out<<4) | (last()<<3) | opcode; + } + + uint32_t out_opcode_; // 28 bits: out, 1 bit: last, 3 (low) bits: opcode + union { // additional instruction arguments: + uint32_t out1_; // opcode == kInstAlt + // alternate next instruction + + int32_t cap_; // opcode == kInstCapture + // Index of capture register (holds text + // position recorded by capturing parentheses). + // For \n (the submatch for the nth parentheses), + // the left parenthesis captures into register 2*n + // and the right one captures into register 2*n+1. + + int32_t match_id_; // opcode == kInstMatch + // Match ID to identify this match (for re2::Set). + + struct { // opcode == kInstByteRange + uint8_t lo_; // byte range is lo_-hi_ inclusive + uint8_t hi_; // + uint16_t hint_foldcase_; // 15 bits: hint, 1 (low) bit: foldcase + // hint to execution engines: the delta to the + // next instruction (in the current list) worth + // exploring iff this instruction matched; 0 + // means there are no remaining possibilities, + // which is most likely for character classes. + // foldcase: A-Z -> a-z before checking range. + }; + + EmptyOp empty_; // opcode == kInstEmptyWidth + // empty_ is bitwise OR of kEmpty* flags above. + }; + + friend class Compiler; + friend struct PatchList; + friend class Prog; + }; + + // Inst must be trivial so that we can freely clear it with memset(3). + // Arrays of Inst are initialised by copying the initial elements with + // memmove(3) and then clearing any remaining elements with memset(3). + static_assert(std::is_trivial::value, "Inst must be trivial"); + + // Whether to anchor the search. + enum Anchor { + kUnanchored, // match anywhere + kAnchored, // match only starting at beginning of text + }; + + // Kind of match to look for (for anchor != kFullMatch) + // + // kLongestMatch mode finds the overall longest + // match but still makes its submatch choices the way + // Perl would, not in the way prescribed by POSIX. + // The POSIX rules are much more expensive to implement, + // and no one has needed them. + // + // kFullMatch is not strictly necessary -- we could use + // kLongestMatch and then check the length of the match -- but + // the matching code can run faster if it knows to consider only + // full matches. + enum MatchKind { + kFirstMatch, // like Perl, PCRE + kLongestMatch, // like egrep or POSIX + kFullMatch, // match only entire text; implies anchor==kAnchored + kManyMatch // for SearchDFA, records set of matches + }; + + Inst *inst(int id) { return &inst_[id]; } + int start() { return start_; } + int start_unanchored() { return start_unanchored_; } + void set_start(int start) { start_ = start; } + void set_start_unanchored(int start) { start_unanchored_ = start; } + int size() { return size_; } + bool reversed() { return reversed_; } + void set_reversed(bool reversed) { reversed_ = reversed; } + int list_count() { return list_count_; } + int inst_count(InstOp op) { return inst_count_[op]; } + uint16_t* list_heads() { return list_heads_.data(); } + void set_dfa_mem(int64_t dfa_mem) { dfa_mem_ = dfa_mem; } + int64_t dfa_mem() { return dfa_mem_; } + int flags() { return flags_; } + void set_flags(int flags) { flags_ = flags; } + bool anchor_start() { return anchor_start_; } + void set_anchor_start(bool b) { anchor_start_ = b; } + bool anchor_end() { return anchor_end_; } + void set_anchor_end(bool b) { anchor_end_ = b; } + int bytemap_range() { return bytemap_range_; } + const uint8_t* bytemap() { return bytemap_; } + + // Lazily computed. + int first_byte(); + + // Returns string representation of program for debugging. + std::string Dump(); + std::string DumpUnanchored(); + std::string DumpByteMap(); + + // Returns the set of kEmpty flags that are in effect at + // position p within context. + static uint32_t EmptyFlags(const StringPiece& context, const char* p); + + // Returns whether byte c is a word character: ASCII only. + // Used by the implementation of \b and \B. + // This is not right for Unicode, but: + // - it's hard to get right in a byte-at-a-time matching world + // (the DFA has only one-byte lookahead). + // - even if the lookahead were possible, the Progs would be huge. + // This crude approximation is the same one PCRE uses. + static bool IsWordChar(uint8_t c) { + return ('A' <= c && c <= 'Z') || + ('a' <= c && c <= 'z') || + ('0' <= c && c <= '9') || + c == '_'; + } + + // Execution engines. They all search for the regexp (run the prog) + // in text, which is in the larger context (used for ^ $ \b etc). + // Anchor and kind control the kind of search. + // Returns true if match found, false if not. + // If match found, fills match[0..nmatch-1] with submatch info. + // match[0] is overall match, match[1] is first set of parens, etc. + // If a particular submatch is not matched during the regexp match, + // it is set to NULL. + // + // Matching text == StringPiece(NULL, 0) is treated as any other empty + // string, but note that on return, it will not be possible to distinguish + // submatches that matched that empty string from submatches that didn't + // match anything. Either way, match[i] == NULL. + + // Search using NFA: can find submatches but kind of slow. + bool SearchNFA(const StringPiece& text, const StringPiece& context, + Anchor anchor, MatchKind kind, + StringPiece* match, int nmatch); + + // Search using DFA: much faster than NFA but only finds + // end of match and can use a lot more memory. + // Returns whether a match was found. + // If the DFA runs out of memory, sets *failed to true and returns false. + // If matches != NULL and kind == kManyMatch and there is a match, + // SearchDFA fills matches with the match IDs of the final matching state. + bool SearchDFA(const StringPiece& text, const StringPiece& context, + Anchor anchor, MatchKind kind, StringPiece* match0, + bool* failed, SparseSet* matches); + + // The callback issued after building each DFA state with BuildEntireDFA(). + // If next is null, then the memory budget has been exhausted and building + // will halt. Otherwise, the state has been built and next points to an array + // of bytemap_range()+1 slots holding the next states as per the bytemap and + // kByteEndText. The number of the state is implied by the callback sequence: + // the first callback is for state 0, the second callback is for state 1, ... + // match indicates whether the state is a matching state. + using DFAStateCallback = std::function; + + // Build the entire DFA for the given match kind. + // Usually the DFA is built out incrementally, as needed, which + // avoids lots of unnecessary work. + // If cb is not empty, it receives one callback per state built. + // Returns the number of states built. + // FOR TESTING OR EXPERIMENTAL PURPOSES ONLY. + int BuildEntireDFA(MatchKind kind, const DFAStateCallback& cb); + + // Controls whether the DFA should bail out early if the NFA would be faster. + // FOR TESTING ONLY. + static void TEST_dfa_should_bail_when_slow(bool b); + + // Compute bytemap. + void ComputeByteMap(); + + // Computes whether all matches must begin with the same first + // byte, and if so, returns that byte. If not, returns -1. + int ComputeFirstByte(); + + // Run peep-hole optimizer on program. + void Optimize(); + + // One-pass NFA: only correct if IsOnePass() is true, + // but much faster than NFA (competitive with PCRE) + // for those expressions. + bool IsOnePass(); + bool SearchOnePass(const StringPiece& text, const StringPiece& context, + Anchor anchor, MatchKind kind, + StringPiece* match, int nmatch); + + // Bit-state backtracking. Fast on small cases but uses memory + // proportional to the product of the list count and the text size. + bool CanBitState() { return list_heads_.data() != NULL; } + bool SearchBitState(const StringPiece& text, const StringPiece& context, + Anchor anchor, MatchKind kind, + StringPiece* match, int nmatch); + + static const int kMaxOnePassCapture = 5; // $0 through $4 + + // Backtracking search: the gold standard against which the other + // implementations are checked. FOR TESTING ONLY. + // It allocates a ton of memory to avoid running forever. + // It is also recursive, so can't use in production (will overflow stacks). + // The name "Unsafe" here is supposed to be a flag that + // you should not be using this function. + bool UnsafeSearchBacktrack(const StringPiece& text, + const StringPiece& context, + Anchor anchor, MatchKind kind, + StringPiece* match, int nmatch); + + // Computes range for any strings matching regexp. The min and max can in + // some cases be arbitrarily precise, so the caller gets to specify the + // maximum desired length of string returned. + // + // Assuming PossibleMatchRange(&min, &max, N) returns successfully, any + // string s that is an anchored match for this regexp satisfies + // min <= s && s <= max. + // + // Note that PossibleMatchRange() will only consider the first copy of an + // infinitely repeated element (i.e., any regexp element followed by a '*' or + // '+' operator). Regexps with "{N}" constructions are not affected, as those + // do not compile down to infinite repetitions. + // + // Returns true on success, false on error. + bool PossibleMatchRange(std::string* min, std::string* max, int maxlen); + + // EXPERIMENTAL! SUBJECT TO CHANGE! + // Outputs the program fanout into the given sparse array. + void Fanout(SparseArray* fanout); + + // Compiles a collection of regexps to Prog. Each regexp will have + // its own Match instruction recording the index in the output vector. + static Prog* CompileSet(Regexp* re, RE2::Anchor anchor, int64_t max_mem); + + // Flattens the Prog from "tree" form to "list" form. This is an in-place + // operation in the sense that the old instructions are lost. + void Flatten(); + + // Walks the Prog; the "successor roots" or predecessors of the reachable + // instructions are marked in rootmap or predmap/predvec, respectively. + // reachable and stk are preallocated scratch structures. + void MarkSuccessors(SparseArray* rootmap, + SparseArray* predmap, + std::vector>* predvec, + SparseSet* reachable, std::vector* stk); + + // Walks the Prog from the given "root" instruction; the "dominator root" + // of the reachable instructions (if such exists) is marked in rootmap. + // reachable and stk are preallocated scratch structures. + void MarkDominator(int root, SparseArray* rootmap, + SparseArray* predmap, + std::vector>* predvec, + SparseSet* reachable, std::vector* stk); + + // Walks the Prog from the given "root" instruction; the reachable + // instructions are emitted in "list" form and appended to flat. + // reachable and stk are preallocated scratch structures. + void EmitList(int root, SparseArray* rootmap, + std::vector* flat, + SparseSet* reachable, std::vector* stk); + + // Computes hints for ByteRange instructions in [begin, end). + void ComputeHints(std::vector* flat, int begin, int end); + + private: + friend class Compiler; + + DFA* GetDFA(MatchKind kind); + void DeleteDFA(DFA* dfa); + + bool anchor_start_; // regexp has explicit start anchor + bool anchor_end_; // regexp has explicit end anchor + bool reversed_; // whether program runs backward over input + bool did_flatten_; // has Flatten been called? + bool did_onepass_; // has IsOnePass been called? + + int start_; // entry point for program + int start_unanchored_; // unanchored entry point for program + int size_; // number of instructions + int bytemap_range_; // bytemap_[x] < bytemap_range_ + int first_byte_; // required first byte for match, or -1 if none + int flags_; // regexp parse flags + + int list_count_; // count of lists (see above) + int inst_count_[kNumInst]; // count of instructions by opcode + PODArray list_heads_; // sparse array enumerating list heads + // not populated if size_ is overly large + + PODArray inst_; // pointer to instruction array + PODArray onepass_nodes_; // data for OnePass nodes + + int64_t dfa_mem_; // Maximum memory for DFAs. + DFA* dfa_first_; // DFA cached for kFirstMatch/kManyMatch + DFA* dfa_longest_; // DFA cached for kLongestMatch/kFullMatch + + uint8_t bytemap_[256]; // map from input bytes to byte classes + + std::once_flag first_byte_once_; + std::once_flag dfa_first_once_; + std::once_flag dfa_longest_once_; + + Prog(const Prog&) = delete; + Prog& operator=(const Prog&) = delete; +}; + +} // namespace re2 + +#endif // RE2_PROG_H_ diff --git a/extern/re2/re2/re2.cc b/extern/re2/re2/re2.cc new file mode 100644 index 0000000000..a4b499258e --- /dev/null +++ b/extern/re2/re2/re2.cc @@ -0,0 +1,1236 @@ +// Copyright 2003-2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Regular expression interface RE2. +// +// Originally the PCRE C++ wrapper, but adapted to use +// the new automata-based regular expression engines. + +#include "re2/re2.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "util/sparse_array.h" +#include "util/strutil.h" +#include "util/utf.h" +#include "re2/prog.h" +#include "re2/regexp.h" + +namespace re2 { + +// Maximum number of args we can set +static const int kMaxArgs = 16; +static const int kVecSize = 1+kMaxArgs; + +const int RE2::Options::kDefaultMaxMem; // initialized in re2.h + +RE2::Options::Options(RE2::CannedOptions opt) + : encoding_(opt == RE2::Latin1 ? EncodingLatin1 : EncodingUTF8), + posix_syntax_(opt == RE2::POSIX), + longest_match_(opt == RE2::POSIX), + log_errors_(opt != RE2::Quiet), + max_mem_(kDefaultMaxMem), + literal_(false), + never_nl_(false), + dot_nl_(false), + never_capture_(false), + case_sensitive_(true), + perl_classes_(false), + word_boundary_(false), + one_line_(false) { +} + +// static empty objects for use as const references. +// To avoid global constructors, allocated in RE2::Init(). +static const std::string* empty_string; +static const std::map* empty_named_groups; +static const std::map* empty_group_names; + +// Converts from Regexp error code to RE2 error code. +// Maybe some day they will diverge. In any event, this +// hides the existence of Regexp from RE2 users. +static RE2::ErrorCode RegexpErrorToRE2(re2::RegexpStatusCode code) { + switch (code) { + case re2::kRegexpSuccess: + return RE2::NoError; + case re2::kRegexpInternalError: + return RE2::ErrorInternal; + case re2::kRegexpBadEscape: + return RE2::ErrorBadEscape; + case re2::kRegexpBadCharClass: + return RE2::ErrorBadCharClass; + case re2::kRegexpBadCharRange: + return RE2::ErrorBadCharRange; + case re2::kRegexpMissingBracket: + return RE2::ErrorMissingBracket; + case re2::kRegexpMissingParen: + return RE2::ErrorMissingParen; + case re2::kRegexpTrailingBackslash: + return RE2::ErrorTrailingBackslash; + case re2::kRegexpRepeatArgument: + return RE2::ErrorRepeatArgument; + case re2::kRegexpRepeatSize: + return RE2::ErrorRepeatSize; + case re2::kRegexpRepeatOp: + return RE2::ErrorRepeatOp; + case re2::kRegexpBadPerlOp: + return RE2::ErrorBadPerlOp; + case re2::kRegexpBadUTF8: + return RE2::ErrorBadUTF8; + case re2::kRegexpBadNamedCapture: + return RE2::ErrorBadNamedCapture; + } + return RE2::ErrorInternal; +} + +static std::string trunc(const StringPiece& pattern) { + if (pattern.size() < 100) + return std::string(pattern); + return std::string(pattern.substr(0, 100)) + "..."; +} + + +RE2::RE2(const char* pattern) { + Init(pattern, DefaultOptions); +} + +RE2::RE2(const std::string& pattern) { + Init(pattern, DefaultOptions); +} + +RE2::RE2(const StringPiece& pattern) { + Init(pattern, DefaultOptions); +} + +RE2::RE2(const StringPiece& pattern, const Options& options) { + Init(pattern, options); +} + +int RE2::Options::ParseFlags() const { + int flags = Regexp::ClassNL; + switch (encoding()) { + default: + if (log_errors()) + LOG(ERROR) << "Unknown encoding " << encoding(); + break; + case RE2::Options::EncodingUTF8: + break; + case RE2::Options::EncodingLatin1: + flags |= Regexp::Latin1; + break; + } + + if (!posix_syntax()) + flags |= Regexp::LikePerl; + + if (literal()) + flags |= Regexp::Literal; + + if (never_nl()) + flags |= Regexp::NeverNL; + + if (dot_nl()) + flags |= Regexp::DotNL; + + if (never_capture()) + flags |= Regexp::NeverCapture; + + if (!case_sensitive()) + flags |= Regexp::FoldCase; + + if (perl_classes()) + flags |= Regexp::PerlClasses; + + if (word_boundary()) + flags |= Regexp::PerlB; + + if (one_line()) + flags |= Regexp::OneLine; + + return flags; +} + +void RE2::Init(const StringPiece& pattern, const Options& options) { + static std::once_flag empty_once; + std::call_once(empty_once, []() { + empty_string = new std::string; + empty_named_groups = new std::map; + empty_group_names = new std::map; + }); + + pattern_ = std::string(pattern); + options_.Copy(options); + entire_regexp_ = NULL; + suffix_regexp_ = NULL; + prog_ = NULL; + num_captures_ = -1; + rprog_ = NULL; + error_ = empty_string; + error_code_ = NoError; + named_groups_ = NULL; + group_names_ = NULL; + + RegexpStatus status; + entire_regexp_ = Regexp::Parse( + pattern_, + static_cast(options_.ParseFlags()), + &status); + if (entire_regexp_ == NULL) { + if (options_.log_errors()) { + LOG(ERROR) << "Error parsing '" << trunc(pattern_) << "': " + << status.Text(); + } + error_ = new std::string(status.Text()); + error_code_ = RegexpErrorToRE2(status.code()); + error_arg_ = std::string(status.error_arg()); + return; + } + + re2::Regexp* suffix; + if (entire_regexp_->RequiredPrefix(&prefix_, &prefix_foldcase_, &suffix)) + suffix_regexp_ = suffix; + else + suffix_regexp_ = entire_regexp_->Incref(); + + // Two thirds of the memory goes to the forward Prog, + // one third to the reverse prog, because the forward + // Prog has two DFAs but the reverse prog has one. + prog_ = suffix_regexp_->CompileToProg(options_.max_mem()*2/3); + if (prog_ == NULL) { + if (options_.log_errors()) + LOG(ERROR) << "Error compiling '" << trunc(pattern_) << "'"; + error_ = new std::string("pattern too large - compile failed"); + error_code_ = RE2::ErrorPatternTooLarge; + return; + } + + // We used to compute this lazily, but it's used during the + // typical control flow for a match call, so we now compute + // it eagerly, which avoids the overhead of std::once_flag. + num_captures_ = suffix_regexp_->NumCaptures(); + + // Could delay this until the first match call that + // cares about submatch information, but the one-pass + // machine's memory gets cut from the DFA memory budget, + // and that is harder to do if the DFA has already + // been built. + is_one_pass_ = prog_->IsOnePass(); +} + +// Returns rprog_, computing it if needed. +re2::Prog* RE2::ReverseProg() const { + std::call_once(rprog_once_, [](const RE2* re) { + re->rprog_ = + re->suffix_regexp_->CompileToReverseProg(re->options_.max_mem() / 3); + if (re->rprog_ == NULL) { + if (re->options_.log_errors()) + LOG(ERROR) << "Error reverse compiling '" << trunc(re->pattern_) << "'"; + re->error_ = + new std::string("pattern too large - reverse compile failed"); + re->error_code_ = RE2::ErrorPatternTooLarge; + } + }, this); + return rprog_; +} + +RE2::~RE2() { + if (suffix_regexp_) + suffix_regexp_->Decref(); + if (entire_regexp_) + entire_regexp_->Decref(); + delete prog_; + delete rprog_; + if (error_ != empty_string) + delete error_; + if (named_groups_ != NULL && named_groups_ != empty_named_groups) + delete named_groups_; + if (group_names_ != NULL && group_names_ != empty_group_names) + delete group_names_; +} + +int RE2::ProgramSize() const { + if (prog_ == NULL) + return -1; + return prog_->size(); +} + +int RE2::ReverseProgramSize() const { + if (prog_ == NULL) + return -1; + Prog* prog = ReverseProg(); + if (prog == NULL) + return -1; + return prog->size(); +} + +static int Fanout(Prog* prog, std::map* histogram) { + SparseArray fanout(prog->size()); + prog->Fanout(&fanout); + histogram->clear(); + for (SparseArray::iterator i = fanout.begin(); i != fanout.end(); ++i) { + // TODO(junyer): Optimise this? + int bucket = 0; + while (1 << bucket < i->value()) { + bucket++; + } + (*histogram)[bucket]++; + } + return histogram->rbegin()->first; +} + +int RE2::ProgramFanout(std::map* histogram) const { + if (prog_ == NULL) + return -1; + return Fanout(prog_, histogram); +} + +int RE2::ReverseProgramFanout(std::map* histogram) const { + if (prog_ == NULL) + return -1; + Prog* prog = ReverseProg(); + if (prog == NULL) + return -1; + return Fanout(prog, histogram); +} + +// Returns named_groups_, computing it if needed. +const std::map& RE2::NamedCapturingGroups() const { + std::call_once(named_groups_once_, [](const RE2* re) { + if (re->suffix_regexp_ != NULL) + re->named_groups_ = re->suffix_regexp_->NamedCaptures(); + if (re->named_groups_ == NULL) + re->named_groups_ = empty_named_groups; + }, this); + return *named_groups_; +} + +// Returns group_names_, computing it if needed. +const std::map& RE2::CapturingGroupNames() const { + std::call_once(group_names_once_, [](const RE2* re) { + if (re->suffix_regexp_ != NULL) + re->group_names_ = re->suffix_regexp_->CaptureNames(); + if (re->group_names_ == NULL) + re->group_names_ = empty_group_names; + }, this); + return *group_names_; +} + +/***** Convenience interfaces *****/ + +bool RE2::FullMatchN(const StringPiece& text, const RE2& re, + const Arg* const args[], int n) { + return re.DoMatch(text, ANCHOR_BOTH, NULL, args, n); +} + +bool RE2::PartialMatchN(const StringPiece& text, const RE2& re, + const Arg* const args[], int n) { + return re.DoMatch(text, UNANCHORED, NULL, args, n); +} + +bool RE2::ConsumeN(StringPiece* input, const RE2& re, + const Arg* const args[], int n) { + size_t consumed; + if (re.DoMatch(*input, ANCHOR_START, &consumed, args, n)) { + input->remove_prefix(consumed); + return true; + } else { + return false; + } +} + +bool RE2::FindAndConsumeN(StringPiece* input, const RE2& re, + const Arg* const args[], int n) { + size_t consumed; + if (re.DoMatch(*input, UNANCHORED, &consumed, args, n)) { + input->remove_prefix(consumed); + return true; + } else { + return false; + } +} + +bool RE2::Replace(std::string* str, + const RE2& re, + const StringPiece& rewrite) { + StringPiece vec[kVecSize]; + int nvec = 1 + MaxSubmatch(rewrite); + if (nvec > static_cast(arraysize(vec))) + return false; + if (!re.Match(*str, 0, str->size(), UNANCHORED, vec, nvec)) + return false; + + std::string s; + if (!re.Rewrite(&s, rewrite, vec, nvec)) + return false; + + assert(vec[0].begin() >= str->data()); + assert(vec[0].end() <= str->data()+str->size()); + str->replace(vec[0].data() - str->data(), vec[0].size(), s); + return true; +} + +int RE2::GlobalReplace(std::string* str, + const RE2& re, + const StringPiece& rewrite) { + StringPiece vec[kVecSize]; + int nvec = 1 + MaxSubmatch(rewrite); + if (nvec > static_cast(arraysize(vec))) + return false; + + const char* p = str->data(); + const char* ep = p + str->size(); + const char* lastend = NULL; + std::string out; + int count = 0; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + // Iterate just once when fuzzing. Otherwise, we easily get bogged down + // and coverage is unlikely to improve despite significant expense. + while (p == str->data()) { +#else + while (p <= ep) { +#endif + if (!re.Match(*str, static_cast(p - str->data()), + str->size(), UNANCHORED, vec, nvec)) + break; + if (p < vec[0].begin()) + out.append(p, vec[0].begin() - p); + if (vec[0].begin() == lastend && vec[0].size() == 0) { + // Disallow empty match at end of last match: skip ahead. + // + // fullrune() takes int, not ptrdiff_t. However, it just looks + // at the leading byte and treats any length >= 4 the same. + if (re.options().encoding() == RE2::Options::EncodingUTF8 && + fullrune(p, static_cast(std::min(ptrdiff_t{4}, ep - p)))) { + // re is in UTF-8 mode and there is enough left of str + // to allow us to advance by up to UTFmax bytes. + Rune r; + int n = chartorune(&r, p); + // Some copies of chartorune have a bug that accepts + // encodings of values in (10FFFF, 1FFFFF] as valid. + if (r > Runemax) { + n = 1; + r = Runeerror; + } + if (!(n == 1 && r == Runeerror)) { // no decoding error + out.append(p, n); + p += n; + continue; + } + } + // Most likely, re is in Latin-1 mode. If it is in UTF-8 mode, + // we fell through from above and the GIGO principle applies. + if (p < ep) + out.append(p, 1); + p++; + continue; + } + re.Rewrite(&out, rewrite, vec, nvec); + p = vec[0].end(); + lastend = p; + count++; + } + + if (count == 0) + return 0; + + if (p < ep) + out.append(p, ep - p); + using std::swap; + swap(out, *str); + return count; +} + +bool RE2::Extract(const StringPiece& text, + const RE2& re, + const StringPiece& rewrite, + std::string* out) { + StringPiece vec[kVecSize]; + int nvec = 1 + MaxSubmatch(rewrite); + if (nvec > static_cast(arraysize(vec))) + return false; + + if (!re.Match(text, 0, text.size(), UNANCHORED, vec, nvec)) + return false; + + out->clear(); + return re.Rewrite(out, rewrite, vec, nvec); +} + +std::string RE2::QuoteMeta(const StringPiece& unquoted) { + std::string result; + result.reserve(unquoted.size() << 1); + + // Escape any ascii character not in [A-Za-z_0-9]. + // + // Note that it's legal to escape a character even if it has no + // special meaning in a regular expression -- so this function does + // that. (This also makes it identical to the perl function of the + // same name except for the null-character special case; + // see `perldoc -f quotemeta`.) + for (size_t ii = 0; ii < unquoted.size(); ++ii) { + // Note that using 'isalnum' here raises the benchmark time from + // 32ns to 58ns: + if ((unquoted[ii] < 'a' || unquoted[ii] > 'z') && + (unquoted[ii] < 'A' || unquoted[ii] > 'Z') && + (unquoted[ii] < '0' || unquoted[ii] > '9') && + unquoted[ii] != '_' && + // If this is the part of a UTF8 or Latin1 character, we need + // to copy this byte without escaping. Experimentally this is + // what works correctly with the regexp library. + !(unquoted[ii] & 128)) { + if (unquoted[ii] == '\0') { // Special handling for null chars. + // Note that this special handling is not strictly required for RE2, + // but this quoting is required for other regexp libraries such as + // PCRE. + // Can't use "\\0" since the next character might be a digit. + result += "\\x00"; + continue; + } + result += '\\'; + } + result += unquoted[ii]; + } + + return result; +} + +bool RE2::PossibleMatchRange(std::string* min, std::string* max, + int maxlen) const { + if (prog_ == NULL) + return false; + + int n = static_cast(prefix_.size()); + if (n > maxlen) + n = maxlen; + + // Determine initial min max from prefix_ literal. + *min = prefix_.substr(0, n); + *max = prefix_.substr(0, n); + if (prefix_foldcase_) { + // prefix is ASCII lowercase; change *min to uppercase. + for (int i = 0; i < n; i++) { + char& c = (*min)[i]; + if ('a' <= c && c <= 'z') + c += 'A' - 'a'; + } + } + + // Add to prefix min max using PossibleMatchRange on regexp. + std::string dmin, dmax; + maxlen -= n; + if (maxlen > 0 && prog_->PossibleMatchRange(&dmin, &dmax, maxlen)) { + min->append(dmin); + max->append(dmax); + } else if (!max->empty()) { + // prog_->PossibleMatchRange has failed us, + // but we still have useful information from prefix_. + // Round up *max to allow any possible suffix. + PrefixSuccessor(max); + } else { + // Nothing useful. + *min = ""; + *max = ""; + return false; + } + + return true; +} + +// Avoid possible locale nonsense in standard strcasecmp. +// The string a is known to be all lowercase. +static int ascii_strcasecmp(const char* a, const char* b, size_t len) { + const char* ae = a + len; + + for (; a < ae; a++, b++) { + uint8_t x = *a; + uint8_t y = *b; + if ('A' <= y && y <= 'Z') + y += 'a' - 'A'; + if (x != y) + return x - y; + } + return 0; +} + + +/***** Actual matching and rewriting code *****/ + +bool RE2::Match(const StringPiece& text, + size_t startpos, + size_t endpos, + Anchor re_anchor, + StringPiece* submatch, + int nsubmatch) const { + if (!ok()) { + if (options_.log_errors()) + LOG(ERROR) << "Invalid RE2: " << *error_; + return false; + } + + if (startpos > endpos || endpos > text.size()) { + if (options_.log_errors()) + LOG(ERROR) << "RE2: invalid startpos, endpos pair. [" + << "startpos: " << startpos << ", " + << "endpos: " << endpos << ", " + << "text size: " << text.size() << "]"; + return false; + } + + StringPiece subtext = text; + subtext.remove_prefix(startpos); + subtext.remove_suffix(text.size() - endpos); + + // Use DFAs to find exact location of match, filter out non-matches. + + // Don't ask for the location if we won't use it. + // SearchDFA can do extra optimizations in that case. + StringPiece match; + StringPiece* matchp = &match; + if (nsubmatch == 0) + matchp = NULL; + + int ncap = 1 + NumberOfCapturingGroups(); + if (ncap > nsubmatch) + ncap = nsubmatch; + + // If the regexp is anchored explicitly, must not be in middle of text. + if (prog_->anchor_start() && startpos != 0) + return false; + + // If the regexp is anchored explicitly, update re_anchor + // so that we can potentially fall into a faster case below. + if (prog_->anchor_start() && prog_->anchor_end()) + re_anchor = ANCHOR_BOTH; + else if (prog_->anchor_start() && re_anchor != ANCHOR_BOTH) + re_anchor = ANCHOR_START; + + // Check for the required prefix, if any. + size_t prefixlen = 0; + if (!prefix_.empty()) { + if (startpos != 0) + return false; + prefixlen = prefix_.size(); + if (prefixlen > subtext.size()) + return false; + if (prefix_foldcase_) { + if (ascii_strcasecmp(&prefix_[0], subtext.data(), prefixlen) != 0) + return false; + } else { + if (memcmp(&prefix_[0], subtext.data(), prefixlen) != 0) + return false; + } + subtext.remove_prefix(prefixlen); + // If there is a required prefix, the anchor must be at least ANCHOR_START. + if (re_anchor != ANCHOR_BOTH) + re_anchor = ANCHOR_START; + } + + Prog::Anchor anchor = Prog::kUnanchored; + Prog::MatchKind kind = Prog::kFirstMatch; + if (options_.longest_match()) + kind = Prog::kLongestMatch; + bool skipped_test = false; + + bool can_one_pass = (is_one_pass_ && ncap <= Prog::kMaxOnePassCapture); + + // BitState allocates a bitmap of size prog_->list_count() * text.size(). + // It also allocates a stack of 3-word structures which could potentially + // grow as large as prog_->list_count() * text.size(), but in practice is + // much smaller. + const int kMaxBitStateBitmapSize = 256*1024; // bitmap size <= max (bits) + bool can_bit_state = prog_->CanBitState(); + size_t bit_state_text_max = kMaxBitStateBitmapSize / prog_->list_count(); + + bool dfa_failed = false; + switch (re_anchor) { + default: + case UNANCHORED: { + if (!prog_->SearchDFA(subtext, text, anchor, kind, + matchp, &dfa_failed, NULL)) { + if (dfa_failed) { + if (options_.log_errors()) + LOG(ERROR) << "DFA out of memory: size " << prog_->size() << ", " + << "bytemap range " << prog_->bytemap_range() << ", " + << "list count " << prog_->list_count(); + // Fall back to NFA below. + skipped_test = true; + break; + } + return false; + } + if (matchp == NULL) // Matched. Don't care where + return true; + // SearchDFA set match[0].end() but didn't know where the + // match started. Run the regexp backward from match[0].end() + // to find the longest possible match -- that's where it started. + Prog* prog = ReverseProg(); + if (prog == NULL) + return false; + if (!prog->SearchDFA(match, text, Prog::kAnchored, + Prog::kLongestMatch, &match, &dfa_failed, NULL)) { + if (dfa_failed) { + if (options_.log_errors()) + LOG(ERROR) << "DFA out of memory: size " << prog->size() << ", " + << "bytemap range " << prog->bytemap_range() << ", " + << "list count " << prog->list_count(); + // Fall back to NFA below. + skipped_test = true; + break; + } + if (options_.log_errors()) + LOG(ERROR) << "SearchDFA inconsistency"; + return false; + } + break; + } + + case ANCHOR_BOTH: + case ANCHOR_START: + if (re_anchor == ANCHOR_BOTH) + kind = Prog::kFullMatch; + anchor = Prog::kAnchored; + + // If only a small amount of text and need submatch + // information anyway and we're going to use OnePass or BitState + // to get it, we might as well not even bother with the DFA: + // OnePass or BitState will be fast enough. + // On tiny texts, OnePass outruns even the DFA, and + // it doesn't have the shared state and occasional mutex that + // the DFA does. + if (can_one_pass && text.size() <= 4096 && + (ncap > 1 || text.size() <= 8)) { + skipped_test = true; + break; + } + if (can_bit_state && text.size() <= bit_state_text_max && ncap > 1) { + skipped_test = true; + break; + } + if (!prog_->SearchDFA(subtext, text, anchor, kind, + &match, &dfa_failed, NULL)) { + if (dfa_failed) { + if (options_.log_errors()) + LOG(ERROR) << "DFA out of memory: size " << prog_->size() << ", " + << "bytemap range " << prog_->bytemap_range() << ", " + << "list count " << prog_->list_count(); + // Fall back to NFA below. + skipped_test = true; + break; + } + return false; + } + break; + } + + if (!skipped_test && ncap <= 1) { + // We know exactly where it matches. That's enough. + if (ncap == 1) + submatch[0] = match; + } else { + StringPiece subtext1; + if (skipped_test) { + // DFA ran out of memory or was skipped: + // need to search in entire original text. + subtext1 = subtext; + } else { + // DFA found the exact match location: + // let NFA run an anchored, full match search + // to find submatch locations. + subtext1 = match; + anchor = Prog::kAnchored; + kind = Prog::kFullMatch; + } + + if (can_one_pass && anchor != Prog::kUnanchored) { + if (!prog_->SearchOnePass(subtext1, text, anchor, kind, submatch, ncap)) { + if (!skipped_test && options_.log_errors()) + LOG(ERROR) << "SearchOnePass inconsistency"; + return false; + } + } else if (can_bit_state && subtext1.size() <= bit_state_text_max) { + if (!prog_->SearchBitState(subtext1, text, anchor, + kind, submatch, ncap)) { + if (!skipped_test && options_.log_errors()) + LOG(ERROR) << "SearchBitState inconsistency"; + return false; + } + } else { + if (!prog_->SearchNFA(subtext1, text, anchor, kind, submatch, ncap)) { + if (!skipped_test && options_.log_errors()) + LOG(ERROR) << "SearchNFA inconsistency"; + return false; + } + } + } + + // Adjust overall match for required prefix that we stripped off. + if (prefixlen > 0 && nsubmatch > 0) + submatch[0] = StringPiece(submatch[0].data() - prefixlen, + submatch[0].size() + prefixlen); + + // Zero submatches that don't exist in the regexp. + for (int i = ncap; i < nsubmatch; i++) + submatch[i] = StringPiece(); + return true; +} + +// Internal matcher - like Match() but takes Args not StringPieces. +bool RE2::DoMatch(const StringPiece& text, + Anchor re_anchor, + size_t* consumed, + const Arg* const* args, + int n) const { + if (!ok()) { + if (options_.log_errors()) + LOG(ERROR) << "Invalid RE2: " << *error_; + return false; + } + + if (NumberOfCapturingGroups() < n) { + // RE has fewer capturing groups than number of Arg pointers passed in. + return false; + } + + // Count number of capture groups needed. + int nvec; + if (n == 0 && consumed == NULL) + nvec = 0; + else + nvec = n+1; + + StringPiece* vec; + StringPiece stkvec[kVecSize]; + StringPiece* heapvec = NULL; + + if (nvec <= static_cast(arraysize(stkvec))) { + vec = stkvec; + } else { + vec = new StringPiece[nvec]; + heapvec = vec; + } + + if (!Match(text, 0, text.size(), re_anchor, vec, nvec)) { + delete[] heapvec; + return false; + } + + if (consumed != NULL) + *consumed = static_cast(vec[0].end() - text.begin()); + + if (n == 0 || args == NULL) { + // We are not interested in results + delete[] heapvec; + return true; + } + + // If we got here, we must have matched the whole pattern. + for (int i = 0; i < n; i++) { + const StringPiece& s = vec[i+1]; + if (!args[i]->Parse(s.data(), s.size())) { + // TODO: Should we indicate what the error was? + delete[] heapvec; + return false; + } + } + + delete[] heapvec; + return true; +} + +// Checks that the rewrite string is well-formed with respect to this +// regular expression. +bool RE2::CheckRewriteString(const StringPiece& rewrite, + std::string* error) const { + int max_token = -1; + for (const char *s = rewrite.data(), *end = s + rewrite.size(); + s < end; s++) { + int c = *s; + if (c != '\\') { + continue; + } + if (++s == end) { + *error = "Rewrite schema error: '\\' not allowed at end."; + return false; + } + c = *s; + if (c == '\\') { + continue; + } + if (!isdigit(c)) { + *error = "Rewrite schema error: " + "'\\' must be followed by a digit or '\\'."; + return false; + } + int n = (c - '0'); + if (max_token < n) { + max_token = n; + } + } + + if (max_token > NumberOfCapturingGroups()) { + *error = StringPrintf( + "Rewrite schema requests %d matches, but the regexp only has %d " + "parenthesized subexpressions.", + max_token, NumberOfCapturingGroups()); + return false; + } + return true; +} + +// Returns the maximum submatch needed for the rewrite to be done by Replace(). +// E.g. if rewrite == "foo \\2,\\1", returns 2. +int RE2::MaxSubmatch(const StringPiece& rewrite) { + int max = 0; + for (const char *s = rewrite.data(), *end = s + rewrite.size(); + s < end; s++) { + if (*s == '\\') { + s++; + int c = (s < end) ? *s : -1; + if (isdigit(c)) { + int n = (c - '0'); + if (n > max) + max = n; + } + } + } + return max; +} + +// Append the "rewrite" string, with backslash subsitutions from "vec", +// to string "out". +bool RE2::Rewrite(std::string* out, + const StringPiece& rewrite, + const StringPiece* vec, + int veclen) const { + for (const char *s = rewrite.data(), *end = s + rewrite.size(); + s < end; s++) { + if (*s != '\\') { + out->push_back(*s); + continue; + } + s++; + int c = (s < end) ? *s : -1; + if (isdigit(c)) { + int n = (c - '0'); + if (n >= veclen) { + if (options_.log_errors()) { + LOG(ERROR) << "requested group " << n + << " in regexp " << rewrite.data(); + } + return false; + } + StringPiece snip = vec[n]; + if (snip.size() > 0) + out->append(snip.data(), snip.size()); + } else if (c == '\\') { + out->push_back('\\'); + } else { + if (options_.log_errors()) + LOG(ERROR) << "invalid rewrite pattern: " << rewrite.data(); + return false; + } + } + return true; +} + +/***** Parsers for various types *****/ + +bool RE2::Arg::parse_null(const char* str, size_t n, void* dest) { + // We fail if somebody asked us to store into a non-NULL void* pointer + return (dest == NULL); +} + +bool RE2::Arg::parse_string(const char* str, size_t n, void* dest) { + if (dest == NULL) return true; + reinterpret_cast(dest)->assign(str, n); + return true; +} + +bool RE2::Arg::parse_stringpiece(const char* str, size_t n, void* dest) { + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = StringPiece(str, n); + return true; +} + +bool RE2::Arg::parse_char(const char* str, size_t n, void* dest) { + if (n != 1) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = str[0]; + return true; +} + +bool RE2::Arg::parse_schar(const char* str, size_t n, void* dest) { + if (n != 1) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = str[0]; + return true; +} + +bool RE2::Arg::parse_uchar(const char* str, size_t n, void* dest) { + if (n != 1) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = str[0]; + return true; +} + +// Largest number spec that we are willing to parse +static const int kMaxNumberLength = 32; + +// REQUIRES "buf" must have length at least nbuf. +// Copies "str" into "buf" and null-terminates. +// Overwrites *np with the new length. +static const char* TerminateNumber(char* buf, size_t nbuf, const char* str, + size_t* np, bool accept_spaces) { + size_t n = *np; + if (n == 0) return ""; + if (n > 0 && isspace(*str)) { + // We are less forgiving than the strtoxxx() routines and do not + // allow leading spaces. We do allow leading spaces for floats. + if (!accept_spaces) { + return ""; + } + while (n > 0 && isspace(*str)) { + n--; + str++; + } + } + + // Although buf has a fixed maximum size, we can still handle + // arbitrarily large integers correctly by omitting leading zeros. + // (Numbers that are still too long will be out of range.) + // Before deciding whether str is too long, + // remove leading zeros with s/000+/00/. + // Leaving the leading two zeros in place means that + // we don't change 0000x123 (invalid) into 0x123 (valid). + // Skip over leading - before replacing. + bool neg = false; + if (n >= 1 && str[0] == '-') { + neg = true; + n--; + str++; + } + + if (n >= 3 && str[0] == '0' && str[1] == '0') { + while (n >= 3 && str[2] == '0') { + n--; + str++; + } + } + + if (neg) { // make room in buf for - + n++; + str--; + } + + if (n > nbuf-1) return ""; + + memmove(buf, str, n); + if (neg) { + buf[0] = '-'; + } + buf[n] = '\0'; + *np = n; + return buf; +} + +bool RE2::Arg::parse_long_radix(const char* str, + size_t n, + void* dest, + int radix) { + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, sizeof buf, str, &n, false); + char* end; + errno = 0; + long r = strtol(str, &end, radix); + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +bool RE2::Arg::parse_ulong_radix(const char* str, + size_t n, + void* dest, + int radix) { + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, sizeof buf, str, &n, false); + if (str[0] == '-') { + // strtoul() will silently accept negative numbers and parse + // them. This module is more strict and treats them as errors. + return false; + } + + char* end; + errno = 0; + unsigned long r = strtoul(str, &end, radix); + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +bool RE2::Arg::parse_short_radix(const char* str, + size_t n, + void* dest, + int radix) { + long r; + if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse + if ((short)r != r) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = (short)r; + return true; +} + +bool RE2::Arg::parse_ushort_radix(const char* str, + size_t n, + void* dest, + int radix) { + unsigned long r; + if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse + if ((unsigned short)r != r) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = (unsigned short)r; + return true; +} + +bool RE2::Arg::parse_int_radix(const char* str, + size_t n, + void* dest, + int radix) { + long r; + if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse + if ((int)r != r) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = (int)r; + return true; +} + +bool RE2::Arg::parse_uint_radix(const char* str, + size_t n, + void* dest, + int radix) { + unsigned long r; + if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse + if ((unsigned int)r != r) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = (unsigned int)r; + return true; +} + +bool RE2::Arg::parse_longlong_radix(const char* str, + size_t n, + void* dest, + int radix) { + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, sizeof buf, str, &n, false); + char* end; + errno = 0; + long long r = strtoll(str, &end, radix); + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +bool RE2::Arg::parse_ulonglong_radix(const char* str, + size_t n, + void* dest, + int radix) { + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, sizeof buf, str, &n, false); + if (str[0] == '-') { + // strtoull() will silently accept negative numbers and parse + // them. This module is more strict and treats them as errors. + return false; + } + char* end; + errno = 0; + unsigned long long r = strtoull(str, &end, radix); + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +static bool parse_double_float(const char* str, size_t n, bool isfloat, + void* dest) { + if (n == 0) return false; + static const int kMaxLength = 200; + char buf[kMaxLength+1]; + str = TerminateNumber(buf, sizeof buf, str, &n, true); + char* end; + errno = 0; + double r; + if (isfloat) { + r = strtof(str, &end); + } else { + r = strtod(str, &end); + } + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + if (isfloat) { + *(reinterpret_cast(dest)) = (float)r; + } else { + *(reinterpret_cast(dest)) = r; + } + return true; +} + +bool RE2::Arg::parse_double(const char* str, size_t n, void* dest) { + return parse_double_float(str, n, false, dest); +} + +bool RE2::Arg::parse_float(const char* str, size_t n, void* dest) { + return parse_double_float(str, n, true, dest); +} + +#define DEFINE_INTEGER_PARSER(name) \ + bool RE2::Arg::parse_##name(const char* str, size_t n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 10); \ + } \ + bool RE2::Arg::parse_##name##_hex(const char* str, size_t n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 16); \ + } \ + bool RE2::Arg::parse_##name##_octal(const char* str, size_t n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 8); \ + } \ + bool RE2::Arg::parse_##name##_cradix(const char* str, size_t n, \ + void* dest) { \ + return parse_##name##_radix(str, n, dest, 0); \ + } + +DEFINE_INTEGER_PARSER(short); +DEFINE_INTEGER_PARSER(ushort); +DEFINE_INTEGER_PARSER(int); +DEFINE_INTEGER_PARSER(uint); +DEFINE_INTEGER_PARSER(long); +DEFINE_INTEGER_PARSER(ulong); +DEFINE_INTEGER_PARSER(longlong); +DEFINE_INTEGER_PARSER(ulonglong); + +#undef DEFINE_INTEGER_PARSER + +} // namespace re2 diff --git a/extern/re2/re2/re2.h b/extern/re2/re2/re2.h new file mode 100644 index 0000000000..c39589d6d6 --- /dev/null +++ b/extern/re2/re2/re2.h @@ -0,0 +1,959 @@ +// Copyright 2003-2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_RE2_H_ +#define RE2_RE2_H_ + +// C++ interface to the re2 regular-expression library. +// RE2 supports Perl-style regular expressions (with extensions like +// \d, \w, \s, ...). +// +// ----------------------------------------------------------------------- +// REGEXP SYNTAX: +// +// This module uses the re2 library and hence supports +// its syntax for regular expressions, which is similar to Perl's with +// some of the more complicated things thrown away. In particular, +// backreferences and generalized assertions are not available, nor is \Z. +// +// See https://github.com/google/re2/wiki/Syntax for the syntax +// supported by RE2, and a comparison with PCRE and PERL regexps. +// +// For those not familiar with Perl's regular expressions, +// here are some examples of the most commonly used extensions: +// +// "hello (\\w+) world" -- \w matches a "word" character +// "version (\\d+)" -- \d matches a digit +// "hello\\s+world" -- \s matches any whitespace character +// "\\b(\\w+)\\b" -- \b matches non-empty string at word boundary +// "(?i)hello" -- (?i) turns on case-insensitive matching +// "/\\*(.*?)\\*/" -- .*? matches . minimum no. of times possible +// +// ----------------------------------------------------------------------- +// MATCHING INTERFACE: +// +// The "FullMatch" operation checks that supplied text matches a +// supplied pattern exactly. +// +// Example: successful match +// CHECK(RE2::FullMatch("hello", "h.*o")); +// +// Example: unsuccessful match (requires full match): +// CHECK(!RE2::FullMatch("hello", "e")); +// +// ----------------------------------------------------------------------- +// UTF-8 AND THE MATCHING INTERFACE: +// +// By default, the pattern and input text are interpreted as UTF-8. +// The RE2::Latin1 option causes them to be interpreted as Latin-1. +// +// Example: +// CHECK(RE2::FullMatch(utf8_string, RE2(utf8_pattern))); +// CHECK(RE2::FullMatch(latin1_string, RE2(latin1_pattern, RE2::Latin1))); +// +// ----------------------------------------------------------------------- +// MATCHING WITH SUBSTRING EXTRACTION: +// +// You can supply extra pointer arguments to extract matched substrings. +// On match failure, none of the pointees will have been modified. +// On match success, the substrings will be converted (as necessary) and +// their values will be assigned to their pointees until all conversions +// have succeeded or one conversion has failed. +// On conversion failure, the pointees will be in an indeterminate state +// because the caller has no way of knowing which conversion failed. +// However, conversion cannot fail for types like string and StringPiece +// that do not inspect the substring contents. Hence, in the common case +// where all of the pointees are of such types, failure is always due to +// match failure and thus none of the pointees will have been modified. +// +// Example: extracts "ruby" into "s" and 1234 into "i" +// int i; +// std::string s; +// CHECK(RE2::FullMatch("ruby:1234", "(\\w+):(\\d+)", &s, &i)); +// +// Example: fails because string cannot be stored in integer +// CHECK(!RE2::FullMatch("ruby", "(.*)", &i)); +// +// Example: fails because there aren't enough sub-patterns +// CHECK(!RE2::FullMatch("ruby:1234", "\\w+:\\d+", &s)); +// +// Example: does not try to extract any extra sub-patterns +// CHECK(RE2::FullMatch("ruby:1234", "(\\w+):(\\d+)", &s)); +// +// Example: does not try to extract into NULL +// CHECK(RE2::FullMatch("ruby:1234", "(\\w+):(\\d+)", NULL, &i)); +// +// Example: integer overflow causes failure +// CHECK(!RE2::FullMatch("ruby:1234567891234", "\\w+:(\\d+)", &i)); +// +// NOTE(rsc): Asking for substrings slows successful matches quite a bit. +// This may get a little faster in the future, but right now is slower +// than PCRE. On the other hand, failed matches run *very* fast (faster +// than PCRE), as do matches without substring extraction. +// +// ----------------------------------------------------------------------- +// PARTIAL MATCHES +// +// You can use the "PartialMatch" operation when you want the pattern +// to match any substring of the text. +// +// Example: simple search for a string: +// CHECK(RE2::PartialMatch("hello", "ell")); +// +// Example: find first number in a string +// int number; +// CHECK(RE2::PartialMatch("x*100 + 20", "(\\d+)", &number)); +// CHECK_EQ(number, 100); +// +// ----------------------------------------------------------------------- +// PRE-COMPILED REGULAR EXPRESSIONS +// +// RE2 makes it easy to use any string as a regular expression, without +// requiring a separate compilation step. +// +// If speed is of the essence, you can create a pre-compiled "RE2" +// object from the pattern and use it multiple times. If you do so, +// you can typically parse text faster than with sscanf. +// +// Example: precompile pattern for faster matching: +// RE2 pattern("h.*o"); +// while (ReadLine(&str)) { +// if (RE2::FullMatch(str, pattern)) ...; +// } +// +// ----------------------------------------------------------------------- +// SCANNING TEXT INCREMENTALLY +// +// The "Consume" operation may be useful if you want to repeatedly +// match regular expressions at the front of a string and skip over +// them as they match. This requires use of the "StringPiece" type, +// which represents a sub-range of a real string. +// +// Example: read lines of the form "var = value" from a string. +// std::string contents = ...; // Fill string somehow +// StringPiece input(contents); // Wrap a StringPiece around it +// +// std::string var; +// int value; +// while (RE2::Consume(&input, "(\\w+) = (\\d+)\n", &var, &value)) { +// ...; +// } +// +// Each successful call to "Consume" will set "var/value", and also +// advance "input" so it points past the matched text. Note that if the +// regular expression matches an empty string, input will advance +// by 0 bytes. If the regular expression being used might match +// an empty string, the loop body must check for this case and either +// advance the string or break out of the loop. +// +// The "FindAndConsume" operation is similar to "Consume" but does not +// anchor your match at the beginning of the string. For example, you +// could extract all words from a string by repeatedly calling +// RE2::FindAndConsume(&input, "(\\w+)", &word) +// +// ----------------------------------------------------------------------- +// USING VARIABLE NUMBER OF ARGUMENTS +// +// The above operations require you to know the number of arguments +// when you write the code. This is not always possible or easy (for +// example, the regular expression may be calculated at run time). +// You can use the "N" version of the operations when the number of +// match arguments are determined at run time. +// +// Example: +// const RE2::Arg* args[10]; +// int n; +// // ... populate args with pointers to RE2::Arg values ... +// // ... set n to the number of RE2::Arg objects ... +// bool match = RE2::FullMatchN(input, pattern, args, n); +// +// The last statement is equivalent to +// +// bool match = RE2::FullMatch(input, pattern, +// *args[0], *args[1], ..., *args[n - 1]); +// +// ----------------------------------------------------------------------- +// PARSING HEX/OCTAL/C-RADIX NUMBERS +// +// By default, if you pass a pointer to a numeric value, the +// corresponding text is interpreted as a base-10 number. You can +// instead wrap the pointer with a call to one of the operators Hex(), +// Octal(), or CRadix() to interpret the text in another base. The +// CRadix operator interprets C-style "0" (base-8) and "0x" (base-16) +// prefixes, but defaults to base-10. +// +// Example: +// int a, b, c, d; +// CHECK(RE2::FullMatch("100 40 0100 0x40", "(.*) (.*) (.*) (.*)", +// RE2::Octal(&a), RE2::Hex(&b), RE2::CRadix(&c), RE2::CRadix(&d)); +// will leave 64 in a, b, c, and d. + +#include +#include +#include +#include +#include +#include + +#include "re2/stringpiece.h" + +namespace re2 { +class Prog; +class Regexp; +} // namespace re2 + +namespace re2 { + +// Interface for regular expression matching. Also corresponds to a +// pre-compiled regular expression. An "RE2" object is safe for +// concurrent use by multiple threads. +class RE2 { + public: + // We convert user-passed pointers into special Arg objects + class Arg; + class Options; + + // Defined in set.h. + class Set; + + enum ErrorCode { + NoError = 0, + + // Unexpected error + ErrorInternal, + + // Parse errors + ErrorBadEscape, // bad escape sequence + ErrorBadCharClass, // bad character class + ErrorBadCharRange, // bad character class range + ErrorMissingBracket, // missing closing ] + ErrorMissingParen, // missing closing ) + ErrorTrailingBackslash, // trailing \ at end of regexp + ErrorRepeatArgument, // repeat argument missing, e.g. "*" + ErrorRepeatSize, // bad repetition argument + ErrorRepeatOp, // bad repetition operator + ErrorBadPerlOp, // bad perl operator + ErrorBadUTF8, // invalid UTF-8 in regexp + ErrorBadNamedCapture, // bad named capture group + ErrorPatternTooLarge // pattern too large (compile failed) + }; + + // Predefined common options. + // If you need more complicated things, instantiate + // an Option class, possibly passing one of these to + // the Option constructor, change the settings, and pass that + // Option class to the RE2 constructor. + enum CannedOptions { + DefaultOptions = 0, + Latin1, // treat input as Latin-1 (default UTF-8) + POSIX, // POSIX syntax, leftmost-longest match + Quiet // do not log about regexp parse errors + }; + + // Need to have the const char* and const std::string& forms for implicit + // conversions when passing string literals to FullMatch and PartialMatch. + // Otherwise the StringPiece form would be sufficient. +#ifndef SWIG + RE2(const char* pattern); + RE2(const std::string& pattern); +#endif + RE2(const StringPiece& pattern); + RE2(const StringPiece& pattern, const Options& options); + ~RE2(); + + // Returns whether RE2 was created properly. + bool ok() const { return error_code() == NoError; } + + // The string specification for this RE2. E.g. + // RE2 re("ab*c?d+"); + // re.pattern(); // "ab*c?d+" + const std::string& pattern() const { return pattern_; } + + // If RE2 could not be created properly, returns an error string. + // Else returns the empty string. + const std::string& error() const { return *error_; } + + // If RE2 could not be created properly, returns an error code. + // Else returns RE2::NoError (== 0). + ErrorCode error_code() const { return error_code_; } + + // If RE2 could not be created properly, returns the offending + // portion of the regexp. + const std::string& error_arg() const { return error_arg_; } + + // Returns the program size, a very approximate measure of a regexp's "cost". + // Larger numbers are more expensive than smaller numbers. + int ProgramSize() const; + int ReverseProgramSize() const; + + // EXPERIMENTAL! SUBJECT TO CHANGE! + // Outputs the program fanout as a histogram bucketed by powers of 2. + // Returns the number of the largest non-empty bucket. + int ProgramFanout(std::map* histogram) const; + int ReverseProgramFanout(std::map* histogram) const; + + // Returns the underlying Regexp; not for general use. + // Returns entire_regexp_ so that callers don't need + // to know about prefix_ and prefix_foldcase_. + re2::Regexp* Regexp() const { return entire_regexp_; } + + /***** The array-based matching interface ******/ + + // The functions here have names ending in 'N' and are used to implement + // the functions whose names are the prefix before the 'N'. It is sometimes + // useful to invoke them directly, but the syntax is awkward, so the 'N'-less + // versions should be preferred. + static bool FullMatchN(const StringPiece& text, const RE2& re, + const Arg* const args[], int n); + static bool PartialMatchN(const StringPiece& text, const RE2& re, + const Arg* const args[], int n); + static bool ConsumeN(StringPiece* input, const RE2& re, + const Arg* const args[], int n); + static bool FindAndConsumeN(StringPiece* input, const RE2& re, + const Arg* const args[], int n); + +#ifndef SWIG + private: + template + static inline bool Apply(F f, SP sp, const RE2& re) { + return f(sp, re, NULL, 0); + } + + template + static inline bool Apply(F f, SP sp, const RE2& re, const A&... a) { + const Arg* const args[] = {&a...}; + const int n = sizeof...(a); + return f(sp, re, args, n); + } + + public: + // In order to allow FullMatch() et al. to be called with a varying number + // of arguments of varying types, we use two layers of variadic templates. + // The first layer constructs the temporary Arg objects. The second layer + // (above) constructs the array of pointers to the temporary Arg objects. + + /***** The useful part: the matching interface *****/ + + // Matches "text" against "re". If pointer arguments are + // supplied, copies matched sub-patterns into them. + // + // You can pass in a "const char*" or a "std::string" for "text". + // You can pass in a "const char*" or a "std::string" or a "RE2" for "re". + // + // The provided pointer arguments can be pointers to any scalar numeric + // type, or one of: + // std::string (matched piece is copied to string) + // StringPiece (StringPiece is mutated to point to matched piece) + // T (where "bool T::ParseFrom(const char*, size_t)" exists) + // (void*)NULL (the corresponding matched sub-pattern is not copied) + // + // Returns true iff all of the following conditions are satisfied: + // a. "text" matches "re" exactly + // b. The number of matched sub-patterns is >= number of supplied pointers + // c. The "i"th argument has a suitable type for holding the + // string captured as the "i"th sub-pattern. If you pass in + // NULL for the "i"th argument, or pass fewer arguments than + // number of sub-patterns, "i"th captured sub-pattern is + // ignored. + // + // CAVEAT: An optional sub-pattern that does not exist in the + // matched string is assigned the empty string. Therefore, the + // following will return false (because the empty string is not a + // valid number): + // int number; + // RE2::FullMatch("abc", "[a-z]+(\\d+)?", &number); + template + static bool FullMatch(const StringPiece& text, const RE2& re, A&&... a) { + return Apply(FullMatchN, text, re, Arg(std::forward(a))...); + } + + // Exactly like FullMatch(), except that "re" is allowed to match + // a substring of "text". + template + static bool PartialMatch(const StringPiece& text, const RE2& re, A&&... a) { + return Apply(PartialMatchN, text, re, Arg(std::forward(a))...); + } + + // Like FullMatch() and PartialMatch(), except that "re" has to match + // a prefix of the text, and "input" is advanced past the matched + // text. Note: "input" is modified iff this routine returns true + // and "re" matched a non-empty substring of "text". + template + static bool Consume(StringPiece* input, const RE2& re, A&&... a) { + return Apply(ConsumeN, input, re, Arg(std::forward(a))...); + } + + // Like Consume(), but does not anchor the match at the beginning of + // the text. That is, "re" need not start its match at the beginning + // of "input". For example, "FindAndConsume(s, "(\\w+)", &word)" finds + // the next word in "s" and stores it in "word". + template + static bool FindAndConsume(StringPiece* input, const RE2& re, A&&... a) { + return Apply(FindAndConsumeN, input, re, Arg(std::forward(a))...); + } +#endif + + // Replace the first match of "re" in "str" with "rewrite". + // Within "rewrite", backslash-escaped digits (\1 to \9) can be + // used to insert text matching corresponding parenthesized group + // from the pattern. \0 in "rewrite" refers to the entire matching + // text. E.g., + // + // std::string s = "yabba dabba doo"; + // CHECK(RE2::Replace(&s, "b+", "d")); + // + // will leave "s" containing "yada dabba doo" + // + // Returns true if the pattern matches and a replacement occurs, + // false otherwise. + static bool Replace(std::string* str, + const RE2& re, + const StringPiece& rewrite); + + // Like Replace(), except replaces successive non-overlapping occurrences + // of the pattern in the string with the rewrite. E.g. + // + // std::string s = "yabba dabba doo"; + // CHECK(RE2::GlobalReplace(&s, "b+", "d")); + // + // will leave "s" containing "yada dada doo" + // Replacements are not subject to re-matching. + // + // Because GlobalReplace only replaces non-overlapping matches, + // replacing "ana" within "banana" makes only one replacement, not two. + // + // Returns the number of replacements made. + static int GlobalReplace(std::string* str, + const RE2& re, + const StringPiece& rewrite); + + // Like Replace, except that if the pattern matches, "rewrite" + // is copied into "out" with substitutions. The non-matching + // portions of "text" are ignored. + // + // Returns true iff a match occurred and the extraction happened + // successfully; if no match occurs, the string is left unaffected. + // + // REQUIRES: "text" must not alias any part of "*out". + static bool Extract(const StringPiece& text, + const RE2& re, + const StringPiece& rewrite, + std::string* out); + + // Escapes all potentially meaningful regexp characters in + // 'unquoted'. The returned string, used as a regular expression, + // will exactly match the original string. For example, + // 1.5-2.0? + // may become: + // 1\.5\-2\.0\? + static std::string QuoteMeta(const StringPiece& unquoted); + + // Computes range for any strings matching regexp. The min and max can in + // some cases be arbitrarily precise, so the caller gets to specify the + // maximum desired length of string returned. + // + // Assuming PossibleMatchRange(&min, &max, N) returns successfully, any + // string s that is an anchored match for this regexp satisfies + // min <= s && s <= max. + // + // Note that PossibleMatchRange() will only consider the first copy of an + // infinitely repeated element (i.e., any regexp element followed by a '*' or + // '+' operator). Regexps with "{N}" constructions are not affected, as those + // do not compile down to infinite repetitions. + // + // Returns true on success, false on error. + bool PossibleMatchRange(std::string* min, std::string* max, + int maxlen) const; + + // Generic matching interface + + // Type of match. + enum Anchor { + UNANCHORED, // No anchoring + ANCHOR_START, // Anchor at start only + ANCHOR_BOTH // Anchor at start and end + }; + + // Return the number of capturing subpatterns, or -1 if the + // regexp wasn't valid on construction. The overall match ($0) + // does not count: if the regexp is "(a)(b)", returns 2. + int NumberOfCapturingGroups() const { return num_captures_; } + + // Return a map from names to capturing indices. + // The map records the index of the leftmost group + // with the given name. + // Only valid until the re is deleted. + const std::map& NamedCapturingGroups() const; + + // Return a map from capturing indices to names. + // The map has no entries for unnamed groups. + // Only valid until the re is deleted. + const std::map& CapturingGroupNames() const; + + // General matching routine. + // Match against text starting at offset startpos + // and stopping the search at offset endpos. + // Returns true if match found, false if not. + // On a successful match, fills in submatch[] (up to nsubmatch entries) + // with information about submatches. + // I.e. matching RE2("(foo)|(bar)baz") on "barbazbla" will return true, with + // submatch[0] = "barbaz", submatch[1].data() = NULL, submatch[2] = "bar", + // submatch[3].data() = NULL, ..., up to submatch[nsubmatch-1].data() = NULL. + // Caveat: submatch[] may be clobbered even on match failure. + // + // Don't ask for more match information than you will use: + // runs much faster with nsubmatch == 1 than nsubmatch > 1, and + // runs even faster if nsubmatch == 0. + // Doesn't make sense to use nsubmatch > 1 + NumberOfCapturingGroups(), + // but will be handled correctly. + // + // Passing text == StringPiece(NULL, 0) will be handled like any other + // empty string, but note that on return, it will not be possible to tell + // whether submatch i matched the empty string or did not match: + // either way, submatch[i].data() == NULL. + bool Match(const StringPiece& text, + size_t startpos, + size_t endpos, + Anchor re_anchor, + StringPiece* submatch, + int nsubmatch) const; + + // Check that the given rewrite string is suitable for use with this + // regular expression. It checks that: + // * The regular expression has enough parenthesized subexpressions + // to satisfy all of the \N tokens in rewrite + // * The rewrite string doesn't have any syntax errors. E.g., + // '\' followed by anything other than a digit or '\'. + // A true return value guarantees that Replace() and Extract() won't + // fail because of a bad rewrite string. + bool CheckRewriteString(const StringPiece& rewrite, + std::string* error) const; + + // Returns the maximum submatch needed for the rewrite to be done by + // Replace(). E.g. if rewrite == "foo \\2,\\1", returns 2. + static int MaxSubmatch(const StringPiece& rewrite); + + // Append the "rewrite" string, with backslash subsitutions from "vec", + // to string "out". + // Returns true on success. This method can fail because of a malformed + // rewrite string. CheckRewriteString guarantees that the rewrite will + // be sucessful. + bool Rewrite(std::string* out, + const StringPiece& rewrite, + const StringPiece* vec, + int veclen) const; + + // Constructor options + class Options { + public: + // The options are (defaults in parentheses): + // + // utf8 (true) text and pattern are UTF-8; otherwise Latin-1 + // posix_syntax (false) restrict regexps to POSIX egrep syntax + // longest_match (false) search for longest match, not first match + // log_errors (true) log syntax and execution errors to ERROR + // max_mem (see below) approx. max memory footprint of RE2 + // literal (false) interpret string as literal, not regexp + // never_nl (false) never match \n, even if it is in regexp + // dot_nl (false) dot matches everything including new line + // never_capture (false) parse all parens as non-capturing + // case_sensitive (true) match is case-sensitive (regexp can override + // with (?i) unless in posix_syntax mode) + // + // The following options are only consulted when posix_syntax == true. + // When posix_syntax == false, these features are always enabled and + // cannot be turned off; to perform multi-line matching in that case, + // begin the regexp with (?m). + // perl_classes (false) allow Perl's \d \s \w \D \S \W + // word_boundary (false) allow Perl's \b \B (word boundary and not) + // one_line (false) ^ and $ only match beginning and end of text + // + // The max_mem option controls how much memory can be used + // to hold the compiled form of the regexp (the Prog) and + // its cached DFA graphs. Code Search placed limits on the number + // of Prog instructions and DFA states: 10,000 for both. + // In RE2, those limits would translate to about 240 KB per Prog + // and perhaps 2.5 MB per DFA (DFA state sizes vary by regexp; RE2 does a + // better job of keeping them small than Code Search did). + // Each RE2 has two Progs (one forward, one reverse), and each Prog + // can have two DFAs (one first match, one longest match). + // That makes 4 DFAs: + // + // forward, first-match - used for UNANCHORED or ANCHOR_START searches + // if opt.longest_match() == false + // forward, longest-match - used for all ANCHOR_BOTH searches, + // and the other two kinds if + // opt.longest_match() == true + // reverse, first-match - never used + // reverse, longest-match - used as second phase for unanchored searches + // + // The RE2 memory budget is statically divided between the two + // Progs and then the DFAs: two thirds to the forward Prog + // and one third to the reverse Prog. The forward Prog gives half + // of what it has left over to each of its DFAs. The reverse Prog + // gives it all to its longest-match DFA. + // + // Once a DFA fills its budget, it flushes its cache and starts over. + // If this happens too often, RE2 falls back on the NFA implementation. + + // For now, make the default budget something close to Code Search. + static const int kDefaultMaxMem = 8<<20; + + enum Encoding { + EncodingUTF8 = 1, + EncodingLatin1 + }; + + Options() : + encoding_(EncodingUTF8), + posix_syntax_(false), + longest_match_(false), + log_errors_(true), + max_mem_(kDefaultMaxMem), + literal_(false), + never_nl_(false), + dot_nl_(false), + never_capture_(false), + case_sensitive_(true), + perl_classes_(false), + word_boundary_(false), + one_line_(false) { + } + + /*implicit*/ Options(CannedOptions); + + Encoding encoding() const { return encoding_; } + void set_encoding(Encoding encoding) { encoding_ = encoding; } + + // Legacy interface to encoding. + // TODO(rsc): Remove once clients have been converted. + bool utf8() const { return encoding_ == EncodingUTF8; } + void set_utf8(bool b) { + if (b) { + encoding_ = EncodingUTF8; + } else { + encoding_ = EncodingLatin1; + } + } + + bool posix_syntax() const { return posix_syntax_; } + void set_posix_syntax(bool b) { posix_syntax_ = b; } + + bool longest_match() const { return longest_match_; } + void set_longest_match(bool b) { longest_match_ = b; } + + bool log_errors() const { return log_errors_; } + void set_log_errors(bool b) { log_errors_ = b; } + + int64_t max_mem() const { return max_mem_; } + void set_max_mem(int64_t m) { max_mem_ = m; } + + bool literal() const { return literal_; } + void set_literal(bool b) { literal_ = b; } + + bool never_nl() const { return never_nl_; } + void set_never_nl(bool b) { never_nl_ = b; } + + bool dot_nl() const { return dot_nl_; } + void set_dot_nl(bool b) { dot_nl_ = b; } + + bool never_capture() const { return never_capture_; } + void set_never_capture(bool b) { never_capture_ = b; } + + bool case_sensitive() const { return case_sensitive_; } + void set_case_sensitive(bool b) { case_sensitive_ = b; } + + bool perl_classes() const { return perl_classes_; } + void set_perl_classes(bool b) { perl_classes_ = b; } + + bool word_boundary() const { return word_boundary_; } + void set_word_boundary(bool b) { word_boundary_ = b; } + + bool one_line() const { return one_line_; } + void set_one_line(bool b) { one_line_ = b; } + + void Copy(const Options& src) { + *this = src; + } + + int ParseFlags() const; + + private: + Encoding encoding_; + bool posix_syntax_; + bool longest_match_; + bool log_errors_; + int64_t max_mem_; + bool literal_; + bool never_nl_; + bool dot_nl_; + bool never_capture_; + bool case_sensitive_; + bool perl_classes_; + bool word_boundary_; + bool one_line_; + }; + + // Returns the options set in the constructor. + const Options& options() const { return options_; } + + // Argument converters; see below. + static inline Arg CRadix(short* x); + static inline Arg CRadix(unsigned short* x); + static inline Arg CRadix(int* x); + static inline Arg CRadix(unsigned int* x); + static inline Arg CRadix(long* x); + static inline Arg CRadix(unsigned long* x); + static inline Arg CRadix(long long* x); + static inline Arg CRadix(unsigned long long* x); + + static inline Arg Hex(short* x); + static inline Arg Hex(unsigned short* x); + static inline Arg Hex(int* x); + static inline Arg Hex(unsigned int* x); + static inline Arg Hex(long* x); + static inline Arg Hex(unsigned long* x); + static inline Arg Hex(long long* x); + static inline Arg Hex(unsigned long long* x); + + static inline Arg Octal(short* x); + static inline Arg Octal(unsigned short* x); + static inline Arg Octal(int* x); + static inline Arg Octal(unsigned int* x); + static inline Arg Octal(long* x); + static inline Arg Octal(unsigned long* x); + static inline Arg Octal(long long* x); + static inline Arg Octal(unsigned long long* x); + + private: + void Init(const StringPiece& pattern, const Options& options); + + bool DoMatch(const StringPiece& text, + Anchor re_anchor, + size_t* consumed, + const Arg* const args[], + int n) const; + + re2::Prog* ReverseProg() const; + + std::string pattern_; // string regular expression + Options options_; // option flags + std::string prefix_; // required prefix (before regexp_) + bool prefix_foldcase_; // prefix is ASCII case-insensitive + re2::Regexp* entire_regexp_; // parsed regular expression + re2::Regexp* suffix_regexp_; // parsed regular expression, prefix removed + re2::Prog* prog_; // compiled program for regexp + int num_captures_; // Number of capturing groups + bool is_one_pass_; // can use prog_->SearchOnePass? + + mutable re2::Prog* rprog_; // reverse program for regexp + mutable const std::string* error_; // Error indicator + // (or points to empty string) + mutable ErrorCode error_code_; // Error code + mutable std::string error_arg_; // Fragment of regexp showing error + + // Map from capture names to indices + mutable const std::map* named_groups_; + + // Map from capture indices to names + mutable const std::map* group_names_; + + // Onces for lazy computations. + mutable std::once_flag rprog_once_; + mutable std::once_flag named_groups_once_; + mutable std::once_flag group_names_once_; + + RE2(const RE2&) = delete; + RE2& operator=(const RE2&) = delete; +}; + +/***** Implementation details *****/ + +// Hex/Octal/Binary? + +// Special class for parsing into objects that define a ParseFrom() method +template +class _RE2_MatchObject { + public: + static inline bool Parse(const char* str, size_t n, void* dest) { + if (dest == NULL) return true; + T* object = reinterpret_cast(dest); + return object->ParseFrom(str, n); + } +}; + +class RE2::Arg { + public: + // Empty constructor so we can declare arrays of RE2::Arg + Arg(); + + // Constructor specially designed for NULL arguments + Arg(void*); + Arg(std::nullptr_t); + + typedef bool (*Parser)(const char* str, size_t n, void* dest); + +// Type-specific parsers +#define MAKE_PARSER(type, name) \ + Arg(type* p) : arg_(p), parser_(name) {} \ + Arg(type* p, Parser parser) : arg_(p), parser_(parser) {} + + MAKE_PARSER(char, parse_char) + MAKE_PARSER(signed char, parse_schar) + MAKE_PARSER(unsigned char, parse_uchar) + MAKE_PARSER(float, parse_float) + MAKE_PARSER(double, parse_double) + MAKE_PARSER(std::string, parse_string) + MAKE_PARSER(StringPiece, parse_stringpiece) + + MAKE_PARSER(short, parse_short) + MAKE_PARSER(unsigned short, parse_ushort) + MAKE_PARSER(int, parse_int) + MAKE_PARSER(unsigned int, parse_uint) + MAKE_PARSER(long, parse_long) + MAKE_PARSER(unsigned long, parse_ulong) + MAKE_PARSER(long long, parse_longlong) + MAKE_PARSER(unsigned long long, parse_ulonglong) + +#undef MAKE_PARSER + + // Generic constructor templates + template Arg(T* p) + : arg_(p), parser_(_RE2_MatchObject::Parse) { } + template Arg(T* p, Parser parser) + : arg_(p), parser_(parser) { } + + // Parse the data + bool Parse(const char* str, size_t n) const; + + private: + void* arg_; + Parser parser_; + + static bool parse_null (const char* str, size_t n, void* dest); + static bool parse_char (const char* str, size_t n, void* dest); + static bool parse_schar (const char* str, size_t n, void* dest); + static bool parse_uchar (const char* str, size_t n, void* dest); + static bool parse_float (const char* str, size_t n, void* dest); + static bool parse_double (const char* str, size_t n, void* dest); + static bool parse_string (const char* str, size_t n, void* dest); + static bool parse_stringpiece (const char* str, size_t n, void* dest); + +#define DECLARE_INTEGER_PARSER(name) \ + private: \ + static bool parse_##name(const char* str, size_t n, void* dest); \ + static bool parse_##name##_radix(const char* str, size_t n, void* dest, \ + int radix); \ + \ + public: \ + static bool parse_##name##_hex(const char* str, size_t n, void* dest); \ + static bool parse_##name##_octal(const char* str, size_t n, void* dest); \ + static bool parse_##name##_cradix(const char* str, size_t n, void* dest); + + DECLARE_INTEGER_PARSER(short) + DECLARE_INTEGER_PARSER(ushort) + DECLARE_INTEGER_PARSER(int) + DECLARE_INTEGER_PARSER(uint) + DECLARE_INTEGER_PARSER(long) + DECLARE_INTEGER_PARSER(ulong) + DECLARE_INTEGER_PARSER(longlong) + DECLARE_INTEGER_PARSER(ulonglong) + +#undef DECLARE_INTEGER_PARSER + +}; + +inline RE2::Arg::Arg() : arg_(NULL), parser_(parse_null) { } +inline RE2::Arg::Arg(void* p) : arg_(p), parser_(parse_null) { } +inline RE2::Arg::Arg(std::nullptr_t p) : arg_(p), parser_(parse_null) { } + +inline bool RE2::Arg::Parse(const char* str, size_t n) const { + return (*parser_)(str, n, arg_); +} + +// This part of the parser, appropriate only for ints, deals with bases +#define MAKE_INTEGER_PARSER(type, name) \ + inline RE2::Arg RE2::Hex(type* ptr) { \ + return RE2::Arg(ptr, RE2::Arg::parse_##name##_hex); \ + } \ + inline RE2::Arg RE2::Octal(type* ptr) { \ + return RE2::Arg(ptr, RE2::Arg::parse_##name##_octal); \ + } \ + inline RE2::Arg RE2::CRadix(type* ptr) { \ + return RE2::Arg(ptr, RE2::Arg::parse_##name##_cradix); \ + } + +MAKE_INTEGER_PARSER(short, short) +MAKE_INTEGER_PARSER(unsigned short, ushort) +MAKE_INTEGER_PARSER(int, int) +MAKE_INTEGER_PARSER(unsigned int, uint) +MAKE_INTEGER_PARSER(long, long) +MAKE_INTEGER_PARSER(unsigned long, ulong) +MAKE_INTEGER_PARSER(long long, longlong) +MAKE_INTEGER_PARSER(unsigned long long, ulonglong) + +#undef MAKE_INTEGER_PARSER + +#ifndef SWIG + +// Silence warnings about missing initializers for members of LazyRE2. +// Note that we test for Clang first because it defines __GNUC__ as well. +#if defined(__clang__) +#elif defined(__GNUC__) && __GNUC__ >= 6 +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif + +// Helper for writing global or static RE2s safely. +// Write +// static LazyRE2 re = {".*"}; +// and then use *re instead of writing +// static RE2 re(".*"); +// The former is more careful about multithreaded +// situations than the latter. +// +// N.B. This class never deletes the RE2 object that +// it constructs: that's a feature, so that it can be used +// for global and function static variables. +class LazyRE2 { + private: + struct NoArg {}; + + public: + typedef RE2 element_type; // support std::pointer_traits + + // Constructor omitted to preserve braced initialization in C++98. + + // Pretend to be a pointer to Type (never NULL due to on-demand creation): + RE2& operator*() const { return *get(); } + RE2* operator->() const { return get(); } + + // Named accessor/initializer: + RE2* get() const { + std::call_once(once_, &LazyRE2::Init, this); + return ptr_; + } + + // All data fields must be public to support {"foo"} initialization. + const char* pattern_; + RE2::CannedOptions options_; + NoArg barrier_against_excess_initializers_; + + mutable RE2* ptr_; + mutable std::once_flag once_; + + private: + static void Init(const LazyRE2* lazy_re2) { + lazy_re2->ptr_ = new RE2(lazy_re2->pattern_, lazy_re2->options_); + } + + void operator=(const LazyRE2&); // disallowed +}; +#endif // SWIG + +} // namespace re2 + +using re2::RE2; +using re2::LazyRE2; + +#endif // RE2_RE2_H_ diff --git a/extern/re2/re2/regexp.cc b/extern/re2/re2/regexp.cc new file mode 100644 index 0000000000..7995ffceb3 --- /dev/null +++ b/extern/re2/re2/regexp.cc @@ -0,0 +1,971 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Regular expression representation. +// Tested by parse_test.cc + +#include "re2/regexp.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "util/mutex.h" +#include "util/utf.h" +#include "re2/stringpiece.h" +#include "re2/walker-inl.h" + +namespace re2 { + +// Constructor. Allocates vectors as appropriate for operator. +Regexp::Regexp(RegexpOp op, ParseFlags parse_flags) + : op_(static_cast(op)), + simple_(false), + parse_flags_(static_cast(parse_flags)), + ref_(1), + nsub_(0), + down_(NULL) { + subone_ = NULL; + memset(the_union_, 0, sizeof the_union_); +} + +// Destructor. Assumes already cleaned up children. +// Private: use Decref() instead of delete to destroy Regexps. +// Can't call Decref on the sub-Regexps here because +// that could cause arbitrarily deep recursion, so +// required Decref() to have handled them for us. +Regexp::~Regexp() { + if (nsub_ > 0) + LOG(DFATAL) << "Regexp not destroyed."; + + switch (op_) { + default: + break; + case kRegexpCapture: + delete name_; + break; + case kRegexpLiteralString: + delete[] runes_; + break; + case kRegexpCharClass: + if (cc_) + cc_->Delete(); + delete ccb_; + break; + } +} + +// If it's possible to destroy this regexp without recurring, +// do so and return true. Else return false. +bool Regexp::QuickDestroy() { + if (nsub_ == 0) { + delete this; + return true; + } + return false; +} + +// Lazily allocated. +static Mutex* ref_mutex; +static std::map* ref_map; + +int Regexp::Ref() { + if (ref_ < kMaxRef) + return ref_; + + MutexLock l(ref_mutex); + return (*ref_map)[this]; +} + +// Increments reference count, returns object as convenience. +Regexp* Regexp::Incref() { + if (ref_ >= kMaxRef-1) { + static std::once_flag ref_once; + std::call_once(ref_once, []() { + ref_mutex = new Mutex; + ref_map = new std::map; + }); + + // Store ref count in overflow map. + MutexLock l(ref_mutex); + if (ref_ == kMaxRef) { + // already overflowed + (*ref_map)[this]++; + } else { + // overflowing now + (*ref_map)[this] = kMaxRef; + ref_ = kMaxRef; + } + return this; + } + + ref_++; + return this; +} + +// Decrements reference count and deletes this object if count reaches 0. +void Regexp::Decref() { + if (ref_ == kMaxRef) { + // Ref count is stored in overflow map. + MutexLock l(ref_mutex); + int r = (*ref_map)[this] - 1; + if (r < kMaxRef) { + ref_ = static_cast(r); + ref_map->erase(this); + } else { + (*ref_map)[this] = r; + } + return; + } + ref_--; + if (ref_ == 0) + Destroy(); +} + +// Deletes this object; ref count has count reached 0. +void Regexp::Destroy() { + if (QuickDestroy()) + return; + + // Handle recursive Destroy with explicit stack + // to avoid arbitrarily deep recursion on process stack [sigh]. + down_ = NULL; + Regexp* stack = this; + while (stack != NULL) { + Regexp* re = stack; + stack = re->down_; + if (re->ref_ != 0) + LOG(DFATAL) << "Bad reference count " << re->ref_; + if (re->nsub_ > 0) { + Regexp** subs = re->sub(); + for (int i = 0; i < re->nsub_; i++) { + Regexp* sub = subs[i]; + if (sub == NULL) + continue; + if (sub->ref_ == kMaxRef) + sub->Decref(); + else + --sub->ref_; + if (sub->ref_ == 0 && !sub->QuickDestroy()) { + sub->down_ = stack; + stack = sub; + } + } + if (re->nsub_ > 1) + delete[] subs; + re->nsub_ = 0; + } + delete re; + } +} + +void Regexp::AddRuneToString(Rune r) { + DCHECK(op_ == kRegexpLiteralString); + if (nrunes_ == 0) { + // start with 8 + runes_ = new Rune[8]; + } else if (nrunes_ >= 8 && (nrunes_ & (nrunes_ - 1)) == 0) { + // double on powers of two + Rune *old = runes_; + runes_ = new Rune[nrunes_ * 2]; + for (int i = 0; i < nrunes_; i++) + runes_[i] = old[i]; + delete[] old; + } + + runes_[nrunes_++] = r; +} + +Regexp* Regexp::HaveMatch(int match_id, ParseFlags flags) { + Regexp* re = new Regexp(kRegexpHaveMatch, flags); + re->match_id_ = match_id; + return re; +} + +Regexp* Regexp::StarPlusOrQuest(RegexpOp op, Regexp* sub, ParseFlags flags) { + // Squash **, ++ and ??. + if (op == sub->op() && flags == sub->parse_flags()) + return sub; + + // Squash *+, *?, +*, +?, ?* and ?+. They all squash to *, so because + // op is Star/Plus/Quest, we just have to check that sub->op() is too. + if ((sub->op() == kRegexpStar || + sub->op() == kRegexpPlus || + sub->op() == kRegexpQuest) && + flags == sub->parse_flags()) { + // If sub is Star, no need to rewrite it. + if (sub->op() == kRegexpStar) + return sub; + + // Rewrite sub to Star. + Regexp* re = new Regexp(kRegexpStar, flags); + re->AllocSub(1); + re->sub()[0] = sub->sub()[0]->Incref(); + sub->Decref(); // We didn't consume the reference after all. + return re; + } + + Regexp* re = new Regexp(op, flags); + re->AllocSub(1); + re->sub()[0] = sub; + return re; +} + +Regexp* Regexp::Plus(Regexp* sub, ParseFlags flags) { + return StarPlusOrQuest(kRegexpPlus, sub, flags); +} + +Regexp* Regexp::Star(Regexp* sub, ParseFlags flags) { + return StarPlusOrQuest(kRegexpStar, sub, flags); +} + +Regexp* Regexp::Quest(Regexp* sub, ParseFlags flags) { + return StarPlusOrQuest(kRegexpQuest, sub, flags); +} + +Regexp* Regexp::ConcatOrAlternate(RegexpOp op, Regexp** sub, int nsub, + ParseFlags flags, bool can_factor) { + if (nsub == 1) + return sub[0]; + + if (nsub == 0) { + if (op == kRegexpAlternate) + return new Regexp(kRegexpNoMatch, flags); + else + return new Regexp(kRegexpEmptyMatch, flags); + } + + Regexp** subcopy = NULL; + if (op == kRegexpAlternate && can_factor) { + // Going to edit sub; make a copy so we don't step on caller. + subcopy = new Regexp*[nsub]; + memmove(subcopy, sub, nsub * sizeof sub[0]); + sub = subcopy; + nsub = FactorAlternation(sub, nsub, flags); + if (nsub == 1) { + Regexp* re = sub[0]; + delete[] subcopy; + return re; + } + } + + if (nsub > kMaxNsub) { + // Too many subexpressions to fit in a single Regexp. + // Make a two-level tree. Two levels gets us to 65535^2. + int nbigsub = (nsub+kMaxNsub-1)/kMaxNsub; + Regexp* re = new Regexp(op, flags); + re->AllocSub(nbigsub); + Regexp** subs = re->sub(); + for (int i = 0; i < nbigsub - 1; i++) + subs[i] = ConcatOrAlternate(op, sub+i*kMaxNsub, kMaxNsub, flags, false); + subs[nbigsub - 1] = ConcatOrAlternate(op, sub+(nbigsub-1)*kMaxNsub, + nsub - (nbigsub-1)*kMaxNsub, flags, + false); + delete[] subcopy; + return re; + } + + Regexp* re = new Regexp(op, flags); + re->AllocSub(nsub); + Regexp** subs = re->sub(); + for (int i = 0; i < nsub; i++) + subs[i] = sub[i]; + + delete[] subcopy; + return re; +} + +Regexp* Regexp::Concat(Regexp** sub, int nsub, ParseFlags flags) { + return ConcatOrAlternate(kRegexpConcat, sub, nsub, flags, false); +} + +Regexp* Regexp::Alternate(Regexp** sub, int nsub, ParseFlags flags) { + return ConcatOrAlternate(kRegexpAlternate, sub, nsub, flags, true); +} + +Regexp* Regexp::AlternateNoFactor(Regexp** sub, int nsub, ParseFlags flags) { + return ConcatOrAlternate(kRegexpAlternate, sub, nsub, flags, false); +} + +Regexp* Regexp::Capture(Regexp* sub, ParseFlags flags, int cap) { + Regexp* re = new Regexp(kRegexpCapture, flags); + re->AllocSub(1); + re->sub()[0] = sub; + re->cap_ = cap; + return re; +} + +Regexp* Regexp::Repeat(Regexp* sub, ParseFlags flags, int min, int max) { + Regexp* re = new Regexp(kRegexpRepeat, flags); + re->AllocSub(1); + re->sub()[0] = sub; + re->min_ = min; + re->max_ = max; + return re; +} + +Regexp* Regexp::NewLiteral(Rune rune, ParseFlags flags) { + Regexp* re = new Regexp(kRegexpLiteral, flags); + re->rune_ = rune; + return re; +} + +Regexp* Regexp::LiteralString(Rune* runes, int nrunes, ParseFlags flags) { + if (nrunes <= 0) + return new Regexp(kRegexpEmptyMatch, flags); + if (nrunes == 1) + return NewLiteral(runes[0], flags); + Regexp* re = new Regexp(kRegexpLiteralString, flags); + for (int i = 0; i < nrunes; i++) + re->AddRuneToString(runes[i]); + return re; +} + +Regexp* Regexp::NewCharClass(CharClass* cc, ParseFlags flags) { + Regexp* re = new Regexp(kRegexpCharClass, flags); + re->cc_ = cc; + return re; +} + +void Regexp::Swap(Regexp* that) { + // Regexp is not trivially copyable, so we cannot freely copy it with + // memmove(3), but swapping objects like so is safe for our purposes. + char tmp[sizeof *this]; + void* vthis = reinterpret_cast(this); + void* vthat = reinterpret_cast(that); + memmove(tmp, vthis, sizeof *this); + memmove(vthis, vthat, sizeof *this); + memmove(vthat, tmp, sizeof *this); +} + +// Tests equality of all top-level structure but not subregexps. +static bool TopEqual(Regexp* a, Regexp* b) { + if (a->op() != b->op()) + return false; + + switch (a->op()) { + case kRegexpNoMatch: + case kRegexpEmptyMatch: + case kRegexpAnyChar: + case kRegexpAnyByte: + case kRegexpBeginLine: + case kRegexpEndLine: + case kRegexpWordBoundary: + case kRegexpNoWordBoundary: + case kRegexpBeginText: + return true; + + case kRegexpEndText: + // The parse flags remember whether it's \z or (?-m:$), + // which matters when testing against PCRE. + return ((a->parse_flags() ^ b->parse_flags()) & Regexp::WasDollar) == 0; + + case kRegexpLiteral: + return a->rune() == b->rune() && + ((a->parse_flags() ^ b->parse_flags()) & Regexp::FoldCase) == 0; + + case kRegexpLiteralString: + return a->nrunes() == b->nrunes() && + ((a->parse_flags() ^ b->parse_flags()) & Regexp::FoldCase) == 0 && + memcmp(a->runes(), b->runes(), + a->nrunes() * sizeof a->runes()[0]) == 0; + + case kRegexpAlternate: + case kRegexpConcat: + return a->nsub() == b->nsub(); + + case kRegexpStar: + case kRegexpPlus: + case kRegexpQuest: + return ((a->parse_flags() ^ b->parse_flags()) & Regexp::NonGreedy) == 0; + + case kRegexpRepeat: + return ((a->parse_flags() ^ b->parse_flags()) & Regexp::NonGreedy) == 0 && + a->min() == b->min() && + a->max() == b->max(); + + case kRegexpCapture: + return a->cap() == b->cap() && a->name() == b->name(); + + case kRegexpHaveMatch: + return a->match_id() == b->match_id(); + + case kRegexpCharClass: { + CharClass* acc = a->cc(); + CharClass* bcc = b->cc(); + return acc->size() == bcc->size() && + acc->end() - acc->begin() == bcc->end() - bcc->begin() && + memcmp(acc->begin(), bcc->begin(), + (acc->end() - acc->begin()) * sizeof acc->begin()[0]) == 0; + } + } + + LOG(DFATAL) << "Unexpected op in Regexp::Equal: " << a->op(); + return 0; +} + +bool Regexp::Equal(Regexp* a, Regexp* b) { + if (a == NULL || b == NULL) + return a == b; + + if (!TopEqual(a, b)) + return false; + + // Fast path: + // return without allocating vector if there are no subregexps. + switch (a->op()) { + case kRegexpAlternate: + case kRegexpConcat: + case kRegexpStar: + case kRegexpPlus: + case kRegexpQuest: + case kRegexpRepeat: + case kRegexpCapture: + break; + + default: + return true; + } + + // Committed to doing real work. + // The stack (vector) has pairs of regexps waiting to + // be compared. The regexps are only equal if + // all the pairs end up being equal. + std::vector stk; + + for (;;) { + // Invariant: TopEqual(a, b) == true. + Regexp* a2; + Regexp* b2; + switch (a->op()) { + default: + break; + case kRegexpAlternate: + case kRegexpConcat: + for (int i = 0; i < a->nsub(); i++) { + a2 = a->sub()[i]; + b2 = b->sub()[i]; + if (!TopEqual(a2, b2)) + return false; + stk.push_back(a2); + stk.push_back(b2); + } + break; + + case kRegexpStar: + case kRegexpPlus: + case kRegexpQuest: + case kRegexpRepeat: + case kRegexpCapture: + a2 = a->sub()[0]; + b2 = b->sub()[0]; + if (!TopEqual(a2, b2)) + return false; + // Really: + // stk.push_back(a2); + // stk.push_back(b2); + // break; + // but faster to assign directly and loop. + a = a2; + b = b2; + continue; + } + + size_t n = stk.size(); + if (n == 0) + break; + + DCHECK_GE(n, 2); + a = stk[n-2]; + b = stk[n-1]; + stk.resize(n-2); + } + + return true; +} + +// Keep in sync with enum RegexpStatusCode in regexp.h +static const char *kErrorStrings[] = { + "no error", + "unexpected error", + "invalid escape sequence", + "invalid character class", + "invalid character class range", + "missing ]", + "missing )", + "trailing \\", + "no argument for repetition operator", + "invalid repetition size", + "bad repetition operator", + "invalid perl operator", + "invalid UTF-8", + "invalid named capture group", +}; + +std::string RegexpStatus::CodeText(enum RegexpStatusCode code) { + if (code < 0 || code >= arraysize(kErrorStrings)) + code = kRegexpInternalError; + return kErrorStrings[code]; +} + +std::string RegexpStatus::Text() const { + if (error_arg_.empty()) + return CodeText(code_); + std::string s; + s.append(CodeText(code_)); + s.append(": "); + s.append(error_arg_.data(), error_arg_.size()); + return s; +} + +void RegexpStatus::Copy(const RegexpStatus& status) { + code_ = status.code_; + error_arg_ = status.error_arg_; +} + +typedef int Ignored; // Walker doesn't exist + +// Walker subclass to count capturing parens in regexp. +class NumCapturesWalker : public Regexp::Walker { + public: + NumCapturesWalker() : ncapture_(0) {} + int ncapture() { return ncapture_; } + + virtual Ignored PreVisit(Regexp* re, Ignored ignored, bool* stop) { + if (re->op() == kRegexpCapture) + ncapture_++; + return ignored; + } + virtual Ignored ShortVisit(Regexp* re, Ignored ignored) { + // Should never be called: we use Walk not WalkExponential. + LOG(DFATAL) << "NumCapturesWalker::ShortVisit called"; + return ignored; + } + + private: + int ncapture_; + + NumCapturesWalker(const NumCapturesWalker&) = delete; + NumCapturesWalker& operator=(const NumCapturesWalker&) = delete; +}; + +int Regexp::NumCaptures() { + NumCapturesWalker w; + w.Walk(this, 0); + return w.ncapture(); +} + +// Walker class to build map of named capture groups and their indices. +class NamedCapturesWalker : public Regexp::Walker { + public: + NamedCapturesWalker() : map_(NULL) {} + ~NamedCapturesWalker() { delete map_; } + + std::map* TakeMap() { + std::map* m = map_; + map_ = NULL; + return m; + } + + Ignored PreVisit(Regexp* re, Ignored ignored, bool* stop) { + if (re->op() == kRegexpCapture && re->name() != NULL) { + // Allocate map once we find a name. + if (map_ == NULL) + map_ = new std::map; + + // Record first occurrence of each name. + // (The rule is that if you have the same name + // multiple times, only the leftmost one counts.) + if (map_->find(*re->name()) == map_->end()) + (*map_)[*re->name()] = re->cap(); + } + return ignored; + } + + virtual Ignored ShortVisit(Regexp* re, Ignored ignored) { + // Should never be called: we use Walk not WalkExponential. + LOG(DFATAL) << "NamedCapturesWalker::ShortVisit called"; + return ignored; + } + + private: + std::map* map_; + + NamedCapturesWalker(const NamedCapturesWalker&) = delete; + NamedCapturesWalker& operator=(const NamedCapturesWalker&) = delete; +}; + +std::map* Regexp::NamedCaptures() { + NamedCapturesWalker w; + w.Walk(this, 0); + return w.TakeMap(); +} + +// Walker class to build map from capture group indices to their names. +class CaptureNamesWalker : public Regexp::Walker { + public: + CaptureNamesWalker() : map_(NULL) {} + ~CaptureNamesWalker() { delete map_; } + + std::map* TakeMap() { + std::map* m = map_; + map_ = NULL; + return m; + } + + Ignored PreVisit(Regexp* re, Ignored ignored, bool* stop) { + if (re->op() == kRegexpCapture && re->name() != NULL) { + // Allocate map once we find a name. + if (map_ == NULL) + map_ = new std::map; + + (*map_)[re->cap()] = *re->name(); + } + return ignored; + } + + virtual Ignored ShortVisit(Regexp* re, Ignored ignored) { + // Should never be called: we use Walk not WalkExponential. + LOG(DFATAL) << "CaptureNamesWalker::ShortVisit called"; + return ignored; + } + + private: + std::map* map_; + + CaptureNamesWalker(const CaptureNamesWalker&) = delete; + CaptureNamesWalker& operator=(const CaptureNamesWalker&) = delete; +}; + +std::map* Regexp::CaptureNames() { + CaptureNamesWalker w; + w.Walk(this, 0); + return w.TakeMap(); +} + +// Determines whether regexp matches must be anchored +// with a fixed string prefix. If so, returns the prefix and +// the regexp that remains after the prefix. The prefix might +// be ASCII case-insensitive. +bool Regexp::RequiredPrefix(std::string* prefix, bool* foldcase, + Regexp** suffix) { + // No need for a walker: the regexp must be of the form + // 1. some number of ^ anchors + // 2. a literal char or string + // 3. the rest + prefix->clear(); + *foldcase = false; + *suffix = NULL; + if (op_ != kRegexpConcat) + return false; + + // Some number of anchors, then a literal or concatenation. + int i = 0; + Regexp** sub = this->sub(); + while (i < nsub_ && sub[i]->op_ == kRegexpBeginText) + i++; + if (i == 0 || i >= nsub_) + return false; + + Regexp* re = sub[i]; + switch (re->op_) { + default: + return false; + + case kRegexpLiteralString: + // Convert to string in proper encoding. + if (re->parse_flags() & Latin1) { + prefix->resize(re->nrunes_); + for (int j = 0; j < re->nrunes_; j++) + (*prefix)[j] = static_cast(re->runes_[j]); + } else { + // Convert to UTF-8 in place. + // Assume worst-case space and then trim. + prefix->resize(re->nrunes_ * UTFmax); + char *p = &(*prefix)[0]; + for (int j = 0; j < re->nrunes_; j++) { + Rune r = re->runes_[j]; + if (r < Runeself) + *p++ = static_cast(r); + else + p += runetochar(p, &r); + } + prefix->resize(p - &(*prefix)[0]); + } + break; + + case kRegexpLiteral: + if ((re->parse_flags() & Latin1) || re->rune_ < Runeself) { + prefix->append(1, static_cast(re->rune_)); + } else { + char buf[UTFmax]; + prefix->append(buf, runetochar(buf, &re->rune_)); + } + break; + } + *foldcase = (sub[i]->parse_flags() & FoldCase) != 0; + i++; + + // The rest. + if (i < nsub_) { + for (int j = i; j < nsub_; j++) + sub[j]->Incref(); + re = Concat(sub + i, nsub_ - i, parse_flags()); + } else { + re = new Regexp(kRegexpEmptyMatch, parse_flags()); + } + *suffix = re; + return true; +} + +// Character class builder is a balanced binary tree (STL set) +// containing non-overlapping, non-abutting RuneRanges. +// The less-than operator used in the tree treats two +// ranges as equal if they overlap at all, so that +// lookups for a particular Rune are possible. + +CharClassBuilder::CharClassBuilder() { + nrunes_ = 0; + upper_ = 0; + lower_ = 0; +} + +// Add lo-hi to the class; return whether class got bigger. +bool CharClassBuilder::AddRange(Rune lo, Rune hi) { + if (hi < lo) + return false; + + if (lo <= 'z' && hi >= 'A') { + // Overlaps some alpha, maybe not all. + // Update bitmaps telling which ASCII letters are in the set. + Rune lo1 = std::max(lo, 'A'); + Rune hi1 = std::min(hi, 'Z'); + if (lo1 <= hi1) + upper_ |= ((1 << (hi1 - lo1 + 1)) - 1) << (lo1 - 'A'); + + lo1 = std::max(lo, 'a'); + hi1 = std::min(hi, 'z'); + if (lo1 <= hi1) + lower_ |= ((1 << (hi1 - lo1 + 1)) - 1) << (lo1 - 'a'); + } + + { // Check whether lo, hi is already in the class. + iterator it = ranges_.find(RuneRange(lo, lo)); + if (it != end() && it->lo <= lo && hi <= it->hi) + return false; + } + + // Look for a range abutting lo on the left. + // If it exists, take it out and increase our range. + if (lo > 0) { + iterator it = ranges_.find(RuneRange(lo-1, lo-1)); + if (it != end()) { + lo = it->lo; + if (it->hi > hi) + hi = it->hi; + nrunes_ -= it->hi - it->lo + 1; + ranges_.erase(it); + } + } + + // Look for a range abutting hi on the right. + // If it exists, take it out and increase our range. + if (hi < Runemax) { + iterator it = ranges_.find(RuneRange(hi+1, hi+1)); + if (it != end()) { + hi = it->hi; + nrunes_ -= it->hi - it->lo + 1; + ranges_.erase(it); + } + } + + // Look for ranges between lo and hi. Take them out. + // This is only safe because the set has no overlapping ranges. + // We've already removed any ranges abutting lo and hi, so + // any that overlap [lo, hi] must be contained within it. + for (;;) { + iterator it = ranges_.find(RuneRange(lo, hi)); + if (it == end()) + break; + nrunes_ -= it->hi - it->lo + 1; + ranges_.erase(it); + } + + // Finally, add [lo, hi]. + nrunes_ += hi - lo + 1; + ranges_.insert(RuneRange(lo, hi)); + return true; +} + +void CharClassBuilder::AddCharClass(CharClassBuilder *cc) { + for (iterator it = cc->begin(); it != cc->end(); ++it) + AddRange(it->lo, it->hi); +} + +bool CharClassBuilder::Contains(Rune r) { + return ranges_.find(RuneRange(r, r)) != end(); +} + +// Does the character class behave the same on A-Z as on a-z? +bool CharClassBuilder::FoldsASCII() { + return ((upper_ ^ lower_) & AlphaMask) == 0; +} + +CharClassBuilder* CharClassBuilder::Copy() { + CharClassBuilder* cc = new CharClassBuilder; + for (iterator it = begin(); it != end(); ++it) + cc->ranges_.insert(RuneRange(it->lo, it->hi)); + cc->upper_ = upper_; + cc->lower_ = lower_; + cc->nrunes_ = nrunes_; + return cc; +} + + + +void CharClassBuilder::RemoveAbove(Rune r) { + if (r >= Runemax) + return; + + if (r < 'z') { + if (r < 'a') + lower_ = 0; + else + lower_ &= AlphaMask >> ('z' - r); + } + + if (r < 'Z') { + if (r < 'A') + upper_ = 0; + else + upper_ &= AlphaMask >> ('Z' - r); + } + + for (;;) { + + iterator it = ranges_.find(RuneRange(r + 1, Runemax)); + if (it == end()) + break; + RuneRange rr = *it; + ranges_.erase(it); + nrunes_ -= rr.hi - rr.lo + 1; + if (rr.lo <= r) { + rr.hi = r; + ranges_.insert(rr); + nrunes_ += rr.hi - rr.lo + 1; + } + } +} + +void CharClassBuilder::Negate() { + // Build up negation and then copy in. + // Could edit ranges in place, but C++ won't let me. + std::vector v; + v.reserve(ranges_.size() + 1); + + // In negation, first range begins at 0, unless + // the current class begins at 0. + iterator it = begin(); + if (it == end()) { + v.push_back(RuneRange(0, Runemax)); + } else { + int nextlo = 0; + if (it->lo == 0) { + nextlo = it->hi + 1; + ++it; + } + for (; it != end(); ++it) { + v.push_back(RuneRange(nextlo, it->lo - 1)); + nextlo = it->hi + 1; + } + if (nextlo <= Runemax) + v.push_back(RuneRange(nextlo, Runemax)); + } + + ranges_.clear(); + for (size_t i = 0; i < v.size(); i++) + ranges_.insert(v[i]); + + upper_ = AlphaMask & ~upper_; + lower_ = AlphaMask & ~lower_; + nrunes_ = Runemax+1 - nrunes_; +} + +// Character class is a sorted list of ranges. +// The ranges are allocated in the same block as the header, +// necessitating a special allocator and Delete method. + +CharClass* CharClass::New(int maxranges) { + CharClass* cc; + uint8_t* data = new uint8_t[sizeof *cc + maxranges*sizeof cc->ranges_[0]]; + cc = reinterpret_cast(data); + cc->ranges_ = reinterpret_cast(data + sizeof *cc); + cc->nranges_ = 0; + cc->folds_ascii_ = false; + cc->nrunes_ = 0; + return cc; +} + +void CharClass::Delete() { + uint8_t* data = reinterpret_cast(this); + delete[] data; +} + +CharClass* CharClass::Negate() { + CharClass* cc = CharClass::New(nranges_+1); + cc->folds_ascii_ = folds_ascii_; + cc->nrunes_ = Runemax + 1 - nrunes_; + int n = 0; + int nextlo = 0; + for (CharClass::iterator it = begin(); it != end(); ++it) { + if (it->lo == nextlo) { + nextlo = it->hi + 1; + } else { + cc->ranges_[n++] = RuneRange(nextlo, it->lo - 1); + nextlo = it->hi + 1; + } + } + if (nextlo <= Runemax) + cc->ranges_[n++] = RuneRange(nextlo, Runemax); + cc->nranges_ = n; + return cc; +} + +bool CharClass::Contains(Rune r) { + RuneRange* rr = ranges_; + int n = nranges_; + while (n > 0) { + int m = n/2; + if (rr[m].hi < r) { + rr += m+1; + n -= m+1; + } else if (r < rr[m].lo) { + n = m; + } else { // rr[m].lo <= r && r <= rr[m].hi + return true; + } + } + return false; +} + +CharClass* CharClassBuilder::GetCharClass() { + CharClass* cc = CharClass::New(static_cast(ranges_.size())); + int n = 0; + for (iterator it = begin(); it != end(); ++it) + cc->ranges_[n++] = *it; + cc->nranges_ = n; + DCHECK_LE(n, static_cast(ranges_.size())); + cc->nrunes_ = nrunes_; + cc->folds_ascii_ = FoldsASCII(); + return cc; +} + +} // namespace re2 diff --git a/extern/re2/re2/regexp.h b/extern/re2/re2/regexp.h new file mode 100644 index 0000000000..a5d85c8128 --- /dev/null +++ b/extern/re2/re2/regexp.h @@ -0,0 +1,652 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_REGEXP_H_ +#define RE2_REGEXP_H_ + +// --- SPONSORED LINK -------------------------------------------------- +// If you want to use this library for regular expression matching, +// you should use re2/re2.h, which provides a class RE2 that +// mimics the PCRE interface provided by PCRE's C++ wrappers. +// This header describes the low-level interface used to implement RE2 +// and may change in backwards-incompatible ways from time to time. +// In contrast, RE2's interface will not. +// --------------------------------------------------------------------- + +// Regular expression library: parsing, execution, and manipulation +// of regular expressions. +// +// Any operation that traverses the Regexp structures should be written +// using Regexp::Walker (see walker-inl.h), not recursively, because deeply nested +// regular expressions such as x++++++++++++++++++++... might cause recursive +// traversals to overflow the stack. +// +// It is the caller's responsibility to provide appropriate mutual exclusion +// around manipulation of the regexps. RE2 does this. +// +// PARSING +// +// Regexp::Parse parses regular expressions encoded in UTF-8. +// The default syntax is POSIX extended regular expressions, +// with the following changes: +// +// 1. Backreferences (optional in POSIX EREs) are not supported. +// (Supporting them precludes the use of DFA-based +// matching engines.) +// +// 2. Collating elements and collation classes are not supported. +// (No one has needed or wanted them.) +// +// The exact syntax accepted can be modified by passing flags to +// Regexp::Parse. In particular, many of the basic Perl additions +// are available. The flags are documented below (search for LikePerl). +// +// If parsed with the flag Regexp::Latin1, both the regular expression +// and the input to the matching routines are assumed to be encoded in +// Latin-1, not UTF-8. +// +// EXECUTION +// +// Once Regexp has parsed a regular expression, it provides methods +// to search text using that regular expression. These methods are +// implemented via calling out to other regular expression libraries. +// (Let's call them the sublibraries.) +// +// To call a sublibrary, Regexp does not simply prepare a +// string version of the regular expression and hand it to the +// sublibrary. Instead, Regexp prepares, from its own parsed form, the +// corresponding internal representation used by the sublibrary. +// This has the drawback of needing to know the internal representation +// used by the sublibrary, but it has two important benefits: +// +// 1. The syntax and meaning of regular expressions is guaranteed +// to be that used by Regexp's parser, not the syntax expected +// by the sublibrary. Regexp might accept a restricted or +// expanded syntax for regular expressions as compared with +// the sublibrary. As long as Regexp can translate from its +// internal form into the sublibrary's, clients need not know +// exactly which sublibrary they are using. +// +// 2. The sublibrary parsers are bypassed. For whatever reason, +// sublibrary regular expression parsers often have security +// problems. For example, plan9grep's regular expression parser +// has a buffer overflow in its handling of large character +// classes, and PCRE's parser has had buffer overflow problems +// in the past. Security-team requires sandboxing of sublibrary +// regular expression parsers. Avoiding the sublibrary parsers +// avoids the sandbox. +// +// The execution methods we use now are provided by the compiled form, +// Prog, described in prog.h +// +// MANIPULATION +// +// Unlike other regular expression libraries, Regexp makes its parsed +// form accessible to clients, so that client code can analyze the +// parsed regular expressions. + +#include +#include +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "util/utf.h" +#include "re2/stringpiece.h" + +namespace re2 { + +// Keep in sync with string list kOpcodeNames[] in testing/dump.cc +enum RegexpOp { + // Matches no strings. + kRegexpNoMatch = 1, + + // Matches empty string. + kRegexpEmptyMatch, + + // Matches rune_. + kRegexpLiteral, + + // Matches runes_. + kRegexpLiteralString, + + // Matches concatenation of sub_[0..nsub-1]. + kRegexpConcat, + // Matches union of sub_[0..nsub-1]. + kRegexpAlternate, + + // Matches sub_[0] zero or more times. + kRegexpStar, + // Matches sub_[0] one or more times. + kRegexpPlus, + // Matches sub_[0] zero or one times. + kRegexpQuest, + + // Matches sub_[0] at least min_ times, at most max_ times. + // max_ == -1 means no upper limit. + kRegexpRepeat, + + // Parenthesized (capturing) subexpression. Index is cap_. + // Optionally, capturing name is name_. + kRegexpCapture, + + // Matches any character. + kRegexpAnyChar, + + // Matches any byte [sic]. + kRegexpAnyByte, + + // Matches empty string at beginning of line. + kRegexpBeginLine, + // Matches empty string at end of line. + kRegexpEndLine, + + // Matches word boundary "\b". + kRegexpWordBoundary, + // Matches not-a-word boundary "\B". + kRegexpNoWordBoundary, + + // Matches empty string at beginning of text. + kRegexpBeginText, + // Matches empty string at end of text. + kRegexpEndText, + + // Matches character class given by cc_. + kRegexpCharClass, + + // Forces match of entire expression right now, + // with match ID match_id_ (used by RE2::Set). + kRegexpHaveMatch, + + kMaxRegexpOp = kRegexpHaveMatch, +}; + +// Keep in sync with string list in regexp.cc +enum RegexpStatusCode { + // No error + kRegexpSuccess = 0, + + // Unexpected error + kRegexpInternalError, + + // Parse errors + kRegexpBadEscape, // bad escape sequence + kRegexpBadCharClass, // bad character class + kRegexpBadCharRange, // bad character class range + kRegexpMissingBracket, // missing closing ] + kRegexpMissingParen, // missing closing ) + kRegexpTrailingBackslash, // at end of regexp + kRegexpRepeatArgument, // repeat argument missing, e.g. "*" + kRegexpRepeatSize, // bad repetition argument + kRegexpRepeatOp, // bad repetition operator + kRegexpBadPerlOp, // bad perl operator + kRegexpBadUTF8, // invalid UTF-8 in regexp + kRegexpBadNamedCapture, // bad named capture +}; + +// Error status for certain operations. +class RegexpStatus { + public: + RegexpStatus() : code_(kRegexpSuccess), tmp_(NULL) {} + ~RegexpStatus() { delete tmp_; } + + void set_code(RegexpStatusCode code) { code_ = code; } + void set_error_arg(const StringPiece& error_arg) { error_arg_ = error_arg; } + void set_tmp(std::string* tmp) { delete tmp_; tmp_ = tmp; } + RegexpStatusCode code() const { return code_; } + const StringPiece& error_arg() const { return error_arg_; } + bool ok() const { return code() == kRegexpSuccess; } + + // Copies state from status. + void Copy(const RegexpStatus& status); + + // Returns text equivalent of code, e.g.: + // "Bad character class" + static std::string CodeText(RegexpStatusCode code); + + // Returns text describing error, e.g.: + // "Bad character class: [z-a]" + std::string Text() const; + + private: + RegexpStatusCode code_; // Kind of error + StringPiece error_arg_; // Piece of regexp containing syntax error. + std::string* tmp_; // Temporary storage, possibly where error_arg_ is. + + RegexpStatus(const RegexpStatus&) = delete; + RegexpStatus& operator=(const RegexpStatus&) = delete; +}; + +// Compiled form; see prog.h +class Prog; + +struct RuneRange { + RuneRange() : lo(0), hi(0) { } + RuneRange(int l, int h) : lo(l), hi(h) { } + Rune lo; + Rune hi; +}; + +// Less-than on RuneRanges treats a == b if they overlap at all. +// This lets us look in a set to find the range covering a particular Rune. +struct RuneRangeLess { + bool operator()(const RuneRange& a, const RuneRange& b) const { + return a.hi < b.lo; + } +}; + +class CharClassBuilder; + +class CharClass { + public: + void Delete(); + + typedef RuneRange* iterator; + iterator begin() { return ranges_; } + iterator end() { return ranges_ + nranges_; } + + int size() { return nrunes_; } + bool empty() { return nrunes_ == 0; } + bool full() { return nrunes_ == Runemax+1; } + bool FoldsASCII() { return folds_ascii_; } + + bool Contains(Rune r); + CharClass* Negate(); + + private: + CharClass(); // not implemented + ~CharClass(); // not implemented + static CharClass* New(int maxranges); + + friend class CharClassBuilder; + + bool folds_ascii_; + int nrunes_; + RuneRange *ranges_; + int nranges_; + + CharClass(const CharClass&) = delete; + CharClass& operator=(const CharClass&) = delete; +}; + +class Regexp { + public: + + // Flags for parsing. Can be ORed together. + enum ParseFlags { + NoParseFlags = 0, + FoldCase = 1<<0, // Fold case during matching (case-insensitive). + Literal = 1<<1, // Treat s as literal string instead of a regexp. + ClassNL = 1<<2, // Allow char classes like [^a-z] and \D and \s + // and [[:space:]] to match newline. + DotNL = 1<<3, // Allow . to match newline. + MatchNL = ClassNL | DotNL, + OneLine = 1<<4, // Treat ^ and $ as only matching at beginning and + // end of text, not around embedded newlines. + // (Perl's default) + Latin1 = 1<<5, // Regexp and text are in Latin1, not UTF-8. + NonGreedy = 1<<6, // Repetition operators are non-greedy by default. + PerlClasses = 1<<7, // Allow Perl character classes like \d. + PerlB = 1<<8, // Allow Perl's \b and \B. + PerlX = 1<<9, // Perl extensions: + // non-capturing parens - (?: ) + // non-greedy operators - *? +? ?? {}? + // flag edits - (?i) (?-i) (?i: ) + // i - FoldCase + // m - !OneLine + // s - DotNL + // U - NonGreedy + // line ends: \A \z + // \Q and \E to disable/enable metacharacters + // (?Pexpr) for named captures + // \C to match any single byte + UnicodeGroups = 1<<10, // Allow \p{Han} for Unicode Han group + // and \P{Han} for its negation. + NeverNL = 1<<11, // Never match NL, even if the regexp mentions + // it explicitly. + NeverCapture = 1<<12, // Parse all parens as non-capturing. + + // As close to Perl as we can get. + LikePerl = ClassNL | OneLine | PerlClasses | PerlB | PerlX | + UnicodeGroups, + + // Internal use only. + WasDollar = 1<<13, // on kRegexpEndText: was $ in regexp text + AllParseFlags = (1<<14)-1, + }; + + // Get. No set, Regexps are logically immutable once created. + RegexpOp op() { return static_cast(op_); } + int nsub() { return nsub_; } + bool simple() { return simple_ != 0; } + ParseFlags parse_flags() { return static_cast(parse_flags_); } + int Ref(); // For testing. + + Regexp** sub() { + if(nsub_ <= 1) + return &subone_; + else + return submany_; + } + + int min() { DCHECK_EQ(op_, kRegexpRepeat); return min_; } + int max() { DCHECK_EQ(op_, kRegexpRepeat); return max_; } + Rune rune() { DCHECK_EQ(op_, kRegexpLiteral); return rune_; } + CharClass* cc() { DCHECK_EQ(op_, kRegexpCharClass); return cc_; } + int cap() { DCHECK_EQ(op_, kRegexpCapture); return cap_; } + const std::string* name() { DCHECK_EQ(op_, kRegexpCapture); return name_; } + Rune* runes() { DCHECK_EQ(op_, kRegexpLiteralString); return runes_; } + int nrunes() { DCHECK_EQ(op_, kRegexpLiteralString); return nrunes_; } + int match_id() { DCHECK_EQ(op_, kRegexpHaveMatch); return match_id_; } + + // Increments reference count, returns object as convenience. + Regexp* Incref(); + + // Decrements reference count and deletes this object if count reaches 0. + void Decref(); + + // Parses string s to produce regular expression, returned. + // Caller must release return value with re->Decref(). + // On failure, sets *status (if status != NULL) and returns NULL. + static Regexp* Parse(const StringPiece& s, ParseFlags flags, + RegexpStatus* status); + + // Returns a _new_ simplified version of the current regexp. + // Does not edit the current regexp. + // Caller must release return value with re->Decref(). + // Simplified means that counted repetition has been rewritten + // into simpler terms and all Perl/POSIX features have been + // removed. The result will capture exactly the same + // subexpressions the original did, unless formatted with ToString. + Regexp* Simplify(); + friend class CoalesceWalker; + friend class SimplifyWalker; + + // Parses the regexp src and then simplifies it and sets *dst to the + // string representation of the simplified form. Returns true on success. + // Returns false and sets *status (if status != NULL) on parse error. + static bool SimplifyRegexp(const StringPiece& src, ParseFlags flags, + std::string* dst, RegexpStatus* status); + + // Returns the number of capturing groups in the regexp. + int NumCaptures(); + friend class NumCapturesWalker; + + // Returns a map from names to capturing group indices, + // or NULL if the regexp contains no named capture groups. + // The caller is responsible for deleting the map. + std::map* NamedCaptures(); + + // Returns a map from capturing group indices to capturing group + // names or NULL if the regexp contains no named capture groups. The + // caller is responsible for deleting the map. + std::map* CaptureNames(); + + // Returns a string representation of the current regexp, + // using as few parentheses as possible. + std::string ToString(); + + // Convenience functions. They consume the passed reference, + // so in many cases you should use, e.g., Plus(re->Incref(), flags). + // They do not consume allocated arrays like subs or runes. + static Regexp* Plus(Regexp* sub, ParseFlags flags); + static Regexp* Star(Regexp* sub, ParseFlags flags); + static Regexp* Quest(Regexp* sub, ParseFlags flags); + static Regexp* Concat(Regexp** subs, int nsubs, ParseFlags flags); + static Regexp* Alternate(Regexp** subs, int nsubs, ParseFlags flags); + static Regexp* Capture(Regexp* sub, ParseFlags flags, int cap); + static Regexp* Repeat(Regexp* sub, ParseFlags flags, int min, int max); + static Regexp* NewLiteral(Rune rune, ParseFlags flags); + static Regexp* NewCharClass(CharClass* cc, ParseFlags flags); + static Regexp* LiteralString(Rune* runes, int nrunes, ParseFlags flags); + static Regexp* HaveMatch(int match_id, ParseFlags flags); + + // Like Alternate but does not factor out common prefixes. + static Regexp* AlternateNoFactor(Regexp** subs, int nsubs, ParseFlags flags); + + // Debugging function. Returns string format for regexp + // that makes structure clear. Does NOT use regexp syntax. + std::string Dump(); + + // Helper traversal class, defined fully in walker-inl.h. + template class Walker; + + // Compile to Prog. See prog.h + // Reverse prog expects to be run over text backward. + // Construction and execution of prog will + // stay within approximately max_mem bytes of memory. + // If max_mem <= 0, a reasonable default is used. + Prog* CompileToProg(int64_t max_mem); + Prog* CompileToReverseProg(int64_t max_mem); + + // Whether to expect this library to find exactly the same answer as PCRE + // when running this regexp. Most regexps do mimic PCRE exactly, but a few + // obscure cases behave differently. Technically this is more a property + // of the Prog than the Regexp, but the computation is much easier to do + // on the Regexp. See mimics_pcre.cc for the exact conditions. + bool MimicsPCRE(); + + // Benchmarking function. + void NullWalk(); + + // Whether every match of this regexp must be anchored and + // begin with a non-empty fixed string (perhaps after ASCII + // case-folding). If so, returns the prefix and the sub-regexp that + // follows it. + // Callers should expect *prefix, *foldcase and *suffix to be "zeroed" + // regardless of the return value. + bool RequiredPrefix(std::string* prefix, bool* foldcase, + Regexp** suffix); + + private: + // Constructor allocates vectors as appropriate for operator. + explicit Regexp(RegexpOp op, ParseFlags parse_flags); + + // Use Decref() instead of delete to release Regexps. + // This is private to catch deletes at compile time. + ~Regexp(); + void Destroy(); + bool QuickDestroy(); + + // Helpers for Parse. Listed here so they can edit Regexps. + class ParseState; + + friend class ParseState; + friend bool ParseCharClass(StringPiece* s, Regexp** out_re, + RegexpStatus* status); + + // Helper for testing [sic]. + friend bool RegexpEqualTestingOnly(Regexp*, Regexp*); + + // Computes whether Regexp is already simple. + bool ComputeSimple(); + + // Constructor that generates a Star, Plus or Quest, + // squashing the pair if sub is also a Star, Plus or Quest. + static Regexp* StarPlusOrQuest(RegexpOp op, Regexp* sub, ParseFlags flags); + + // Constructor that generates a concatenation or alternation, + // enforcing the limit on the number of subexpressions for + // a particular Regexp. + static Regexp* ConcatOrAlternate(RegexpOp op, Regexp** subs, int nsubs, + ParseFlags flags, bool can_factor); + + // Returns the leading string that re starts with. + // The returned Rune* points into a piece of re, + // so it must not be used after the caller calls re->Decref(). + static Rune* LeadingString(Regexp* re, int* nrune, ParseFlags* flags); + + // Removes the first n leading runes from the beginning of re. + // Edits re in place. + static void RemoveLeadingString(Regexp* re, int n); + + // Returns the leading regexp in re's top-level concatenation. + // The returned Regexp* points at re or a sub-expression of re, + // so it must not be used after the caller calls re->Decref(). + static Regexp* LeadingRegexp(Regexp* re); + + // Removes LeadingRegexp(re) from re and returns the remainder. + // Might edit re in place. + static Regexp* RemoveLeadingRegexp(Regexp* re); + + // Simplifies an alternation of literal strings by factoring out + // common prefixes. + static int FactorAlternation(Regexp** sub, int nsub, ParseFlags flags); + friend class FactorAlternationImpl; + + // Is a == b? Only efficient on regexps that have not been through + // Simplify yet - the expansion of a kRegexpRepeat will make this + // take a long time. Do not call on such regexps, hence private. + static bool Equal(Regexp* a, Regexp* b); + + // Allocate space for n sub-regexps. + void AllocSub(int n) { + DCHECK(n >= 0 && static_cast(n) == n); + if (n > 1) + submany_ = new Regexp*[n]; + nsub_ = static_cast(n); + } + + // Add Rune to LiteralString + void AddRuneToString(Rune r); + + // Swaps this with that, in place. + void Swap(Regexp *that); + + // Operator. See description of operators above. + // uint8_t instead of RegexpOp to control space usage. + uint8_t op_; + + // Is this regexp structure already simple + // (has it been returned by Simplify)? + // uint8_t instead of bool to control space usage. + uint8_t simple_; + + // Flags saved from parsing and used during execution. + // (Only FoldCase is used.) + // uint16_t instead of ParseFlags to control space usage. + uint16_t parse_flags_; + + // Reference count. Exists so that SimplifyRegexp can build + // regexp structures that are dags rather than trees to avoid + // exponential blowup in space requirements. + // uint16_t to control space usage. + // The standard regexp routines will never generate a + // ref greater than the maximum repeat count (kMaxRepeat), + // but even so, Incref and Decref consult an overflow map + // when ref_ reaches kMaxRef. + uint16_t ref_; + static const uint16_t kMaxRef = 0xffff; + + // Subexpressions. + // uint16_t to control space usage. + // Concat and Alternate handle larger numbers of subexpressions + // by building concatenation or alternation trees. + // Other routines should call Concat or Alternate instead of + // filling in sub() by hand. + uint16_t nsub_; + static const uint16_t kMaxNsub = 0xffff; + union { + Regexp** submany_; // if nsub_ > 1 + Regexp* subone_; // if nsub_ == 1 + }; + + // Extra space for parse and teardown stacks. + Regexp* down_; + + // Arguments to operator. See description of operators above. + union { + struct { // Repeat + int max_; + int min_; + }; + struct { // Capture + int cap_; + std::string* name_; + }; + struct { // LiteralString + int nrunes_; + Rune* runes_; + }; + struct { // CharClass + // These two could be in separate union members, + // but it wouldn't save any space (there are other two-word structs) + // and keeping them separate avoids confusion during parsing. + CharClass* cc_; + CharClassBuilder* ccb_; + }; + Rune rune_; // Literal + int match_id_; // HaveMatch + void *the_union_[2]; // as big as any other element, for memset + }; + + Regexp(const Regexp&) = delete; + Regexp& operator=(const Regexp&) = delete; +}; + +// Character class set: contains non-overlapping, non-abutting RuneRanges. +typedef std::set RuneRangeSet; + +class CharClassBuilder { + public: + CharClassBuilder(); + + typedef RuneRangeSet::iterator iterator; + iterator begin() { return ranges_.begin(); } + iterator end() { return ranges_.end(); } + + int size() { return nrunes_; } + bool empty() { return nrunes_ == 0; } + bool full() { return nrunes_ == Runemax+1; } + + bool Contains(Rune r); + bool FoldsASCII(); + bool AddRange(Rune lo, Rune hi); // returns whether class changed + CharClassBuilder* Copy(); + void AddCharClass(CharClassBuilder* cc); + void Negate(); + void RemoveAbove(Rune r); + CharClass* GetCharClass(); + void AddRangeFlags(Rune lo, Rune hi, Regexp::ParseFlags parse_flags); + + private: + static const uint32_t AlphaMask = (1<<26) - 1; + uint32_t upper_; // bitmap of A-Z + uint32_t lower_; // bitmap of a-z + int nrunes_; + RuneRangeSet ranges_; + + CharClassBuilder(const CharClassBuilder&) = delete; + CharClassBuilder& operator=(const CharClassBuilder&) = delete; +}; + +// Bitwise ops on ParseFlags produce ParseFlags. +inline Regexp::ParseFlags operator|(Regexp::ParseFlags a, + Regexp::ParseFlags b) { + return static_cast( + static_cast(a) | static_cast(b)); +} + +inline Regexp::ParseFlags operator^(Regexp::ParseFlags a, + Regexp::ParseFlags b) { + return static_cast( + static_cast(a) ^ static_cast(b)); +} + +inline Regexp::ParseFlags operator&(Regexp::ParseFlags a, + Regexp::ParseFlags b) { + return static_cast( + static_cast(a) & static_cast(b)); +} + +inline Regexp::ParseFlags operator~(Regexp::ParseFlags a) { + // Attempting to produce a value out of enum's range has undefined behaviour. + return static_cast( + ~static_cast(a) & static_cast(Regexp::AllParseFlags)); +} + +} // namespace re2 + +#endif // RE2_REGEXP_H_ diff --git a/extern/re2/re2/set.cc b/extern/re2/re2/set.cc new file mode 100644 index 0000000000..d4c34ad8f1 --- /dev/null +++ b/extern/re2/re2/set.cc @@ -0,0 +1,153 @@ +// Copyright 2010 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "re2/set.h" + +#include +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "util/pod_array.h" +#include "re2/stringpiece.h" +#include "re2/prog.h" +#include "re2/re2.h" +#include "re2/regexp.h" + +namespace re2 { + +RE2::Set::Set(const RE2::Options& options, RE2::Anchor anchor) { + options_.Copy(options); + options_.set_never_capture(true); // might unblock some optimisations + anchor_ = anchor; + prog_ = NULL; + compiled_ = false; + size_ = 0; +} + +RE2::Set::~Set() { + for (size_t i = 0; i < elem_.size(); i++) + elem_[i].second->Decref(); + delete prog_; +} + +int RE2::Set::Add(const StringPiece& pattern, std::string* error) { + if (compiled_) { + LOG(DFATAL) << "RE2::Set::Add() called after compiling"; + return -1; + } + + Regexp::ParseFlags pf = static_cast( + options_.ParseFlags()); + RegexpStatus status; + re2::Regexp* re = Regexp::Parse(pattern, pf, &status); + if (re == NULL) { + if (error != NULL) + *error = status.Text(); + if (options_.log_errors()) + LOG(ERROR) << "Error parsing '" << pattern << "': " << status.Text(); + return -1; + } + + // Concatenate with match index and push on vector. + int n = static_cast(elem_.size()); + re2::Regexp* m = re2::Regexp::HaveMatch(n, pf); + if (re->op() == kRegexpConcat) { + int nsub = re->nsub(); + PODArray sub(nsub + 1); + for (int i = 0; i < nsub; i++) + sub[i] = re->sub()[i]->Incref(); + sub[nsub] = m; + re->Decref(); + re = re2::Regexp::Concat(sub.data(), nsub + 1, pf); + } else { + re2::Regexp* sub[2]; + sub[0] = re; + sub[1] = m; + re = re2::Regexp::Concat(sub, 2, pf); + } + elem_.emplace_back(std::string(pattern), re); + return n; +} + +bool RE2::Set::Compile() { + if (compiled_) { + LOG(DFATAL) << "RE2::Set::Compile() called more than once"; + return false; + } + compiled_ = true; + size_ = static_cast(elem_.size()); + + // Sort the elements by their patterns. This is good enough for now + // until we have a Regexp comparison function. (Maybe someday...) + std::sort(elem_.begin(), elem_.end(), + [](const Elem& a, const Elem& b) -> bool { + return a.first < b.first; + }); + + PODArray sub(size_); + for (int i = 0; i < size_; i++) + sub[i] = elem_[i].second; + elem_.clear(); + elem_.shrink_to_fit(); + + Regexp::ParseFlags pf = static_cast( + options_.ParseFlags()); + re2::Regexp* re = re2::Regexp::Alternate(sub.data(), size_, pf); + + prog_ = Prog::CompileSet(re, anchor_, options_.max_mem()); + re->Decref(); + return prog_ != NULL; +} + +bool RE2::Set::Match(const StringPiece& text, std::vector* v) const { + return Match(text, v, NULL); +} + +bool RE2::Set::Match(const StringPiece& text, std::vector* v, + ErrorInfo* error_info) const { + if (!compiled_) { + LOG(DFATAL) << "RE2::Set::Match() called before compiling"; + if (error_info != NULL) + error_info->kind = kNotCompiled; + return false; + } + bool dfa_failed = false; + std::unique_ptr matches; + if (v != NULL) { + matches.reset(new SparseSet(size_)); + v->clear(); + } + bool ret = prog_->SearchDFA(text, text, Prog::kAnchored, Prog::kManyMatch, + NULL, &dfa_failed, matches.get()); + if (dfa_failed) { + if (options_.log_errors()) + LOG(ERROR) << "DFA out of memory: size " << prog_->size() << ", " + << "bytemap range " << prog_->bytemap_range() << ", " + << "list count " << prog_->list_count(); + if (error_info != NULL) + error_info->kind = kOutOfMemory; + return false; + } + if (ret == false) { + if (error_info != NULL) + error_info->kind = kNoError; + return false; + } + if (v != NULL) { + if (matches->empty()) { + LOG(DFATAL) << "RE2::Set::Match() matched, but no matches returned?!"; + if (error_info != NULL) + error_info->kind = kInconsistent; + return false; + } + v->assign(matches->begin(), matches->end()); + } + if (error_info != NULL) + error_info->kind = kNoError; + return true; +} + +} // namespace re2 diff --git a/extern/re2/re2/set.h b/extern/re2/re2/set.h new file mode 100644 index 0000000000..59733fd94c --- /dev/null +++ b/extern/re2/re2/set.h @@ -0,0 +1,80 @@ +// Copyright 2010 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_SET_H_ +#define RE2_SET_H_ + +#include +#include +#include + +#include "re2/re2.h" + +namespace re2 { +class Prog; +class Regexp; +} // namespace re2 + +namespace re2 { + +// An RE2::Set represents a collection of regexps that can +// be searched for simultaneously. +class RE2::Set { + public: + enum ErrorKind { + kNoError = 0, + kNotCompiled, // The set is not compiled. + kOutOfMemory, // The DFA ran out of memory. + kInconsistent, // The result is inconsistent. This should never happen. + }; + + struct ErrorInfo { + ErrorKind kind; + }; + + Set(const RE2::Options& options, RE2::Anchor anchor); + ~Set(); + + // Adds pattern to the set using the options passed to the constructor. + // Returns the index that will identify the regexp in the output of Match(), + // or -1 if the regexp cannot be parsed. + // Indices are assigned in sequential order starting from 0. + // Errors do not increment the index; if error is not NULL, *error will hold + // the error message from the parser. + int Add(const StringPiece& pattern, std::string* error); + + // Compiles the set in preparation for matching. + // Returns false if the compiler runs out of memory. + // Add() must not be called again after Compile(). + // Compile() must be called before Match(). + bool Compile(); + + // Returns true if text matches at least one of the regexps in the set. + // Fills v (if not NULL) with the indices of the matching regexps. + // Callers must not expect v to be sorted. + bool Match(const StringPiece& text, std::vector* v) const; + + // As above, but populates error_info (if not NULL) when none of the regexps + // in the set matched. This can inform callers when DFA execution fails, for + // example, because they might wish to handle that case differently. + bool Match(const StringPiece& text, std::vector* v, + ErrorInfo* error_info) const; + + private: + typedef std::pair Elem; + + RE2::Options options_; + RE2::Anchor anchor_; + std::vector elem_; + re2::Prog* prog_; + bool compiled_; + int size_; + + Set(const Set&) = delete; + Set& operator=(const Set&) = delete; +}; + +} // namespace re2 + +#endif // RE2_SET_H_ diff --git a/extern/re2/re2/simplify.cc b/extern/re2/re2/simplify.cc new file mode 100644 index 0000000000..8939678619 --- /dev/null +++ b/extern/re2/re2/simplify.cc @@ -0,0 +1,655 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Rewrite POSIX and other features in re +// to use simple extended regular expression features. +// Also sort and simplify character classes. + +#include + +#include "util/util.h" +#include "util/logging.h" +#include "util/pod_array.h" +#include "util/utf.h" +#include "re2/regexp.h" +#include "re2/walker-inl.h" + +namespace re2 { + +// Parses the regexp src and then simplifies it and sets *dst to the +// string representation of the simplified form. Returns true on success. +// Returns false and sets *error (if error != NULL) on error. +bool Regexp::SimplifyRegexp(const StringPiece& src, ParseFlags flags, + std::string* dst, RegexpStatus* status) { + Regexp* re = Parse(src, flags, status); + if (re == NULL) + return false; + Regexp* sre = re->Simplify(); + re->Decref(); + if (sre == NULL) { + // Should not happen, since Simplify never fails. + LOG(ERROR) << "Simplify failed on " << src; + if (status) { + status->set_code(kRegexpInternalError); + status->set_error_arg(src); + } + return false; + } + *dst = sre->ToString(); + sre->Decref(); + return true; +} + +// Assuming the simple_ flags on the children are accurate, +// is this Regexp* simple? +bool Regexp::ComputeSimple() { + Regexp** subs; + switch (op_) { + case kRegexpNoMatch: + case kRegexpEmptyMatch: + case kRegexpLiteral: + case kRegexpLiteralString: + case kRegexpBeginLine: + case kRegexpEndLine: + case kRegexpBeginText: + case kRegexpWordBoundary: + case kRegexpNoWordBoundary: + case kRegexpEndText: + case kRegexpAnyChar: + case kRegexpAnyByte: + case kRegexpHaveMatch: + return true; + case kRegexpConcat: + case kRegexpAlternate: + // These are simple as long as the subpieces are simple. + subs = sub(); + for (int i = 0; i < nsub_; i++) + if (!subs[i]->simple()) + return false; + return true; + case kRegexpCharClass: + // Simple as long as the char class is not empty, not full. + if (ccb_ != NULL) + return !ccb_->empty() && !ccb_->full(); + return !cc_->empty() && !cc_->full(); + case kRegexpCapture: + subs = sub(); + return subs[0]->simple(); + case kRegexpStar: + case kRegexpPlus: + case kRegexpQuest: + subs = sub(); + if (!subs[0]->simple()) + return false; + switch (subs[0]->op_) { + case kRegexpStar: + case kRegexpPlus: + case kRegexpQuest: + case kRegexpEmptyMatch: + case kRegexpNoMatch: + return false; + default: + break; + } + return true; + case kRegexpRepeat: + return false; + } + LOG(DFATAL) << "Case not handled in ComputeSimple: " << op_; + return false; +} + +// Walker subclass used by Simplify. +// Coalesces runs of star/plus/quest/repeat of the same literal along with any +// occurrences of that literal into repeats of that literal. It also works for +// char classes, any char and any byte. +// PostVisit creates the coalesced result, which should then be simplified. +class CoalesceWalker : public Regexp::Walker { + public: + CoalesceWalker() {} + virtual Regexp* PostVisit(Regexp* re, Regexp* parent_arg, Regexp* pre_arg, + Regexp** child_args, int nchild_args); + virtual Regexp* Copy(Regexp* re); + virtual Regexp* ShortVisit(Regexp* re, Regexp* parent_arg); + + private: + // These functions are declared inside CoalesceWalker so that + // they can edit the private fields of the Regexps they construct. + + // Returns true if r1 and r2 can be coalesced. In particular, ensures that + // the parse flags are consistent. (They will not be checked again later.) + static bool CanCoalesce(Regexp* r1, Regexp* r2); + + // Coalesces *r1ptr and *r2ptr. In most cases, the array elements afterwards + // will be empty match and the coalesced op. In other cases, where part of a + // literal string was removed to be coalesced, the array elements afterwards + // will be the coalesced op and the remainder of the literal string. + static void DoCoalesce(Regexp** r1ptr, Regexp** r2ptr); + + CoalesceWalker(const CoalesceWalker&) = delete; + CoalesceWalker& operator=(const CoalesceWalker&) = delete; +}; + +// Walker subclass used by Simplify. +// The simplify walk is purely post-recursive: given the simplified children, +// PostVisit creates the simplified result. +// The child_args are simplified Regexp*s. +class SimplifyWalker : public Regexp::Walker { + public: + SimplifyWalker() {} + virtual Regexp* PreVisit(Regexp* re, Regexp* parent_arg, bool* stop); + virtual Regexp* PostVisit(Regexp* re, Regexp* parent_arg, Regexp* pre_arg, + Regexp** child_args, int nchild_args); + virtual Regexp* Copy(Regexp* re); + virtual Regexp* ShortVisit(Regexp* re, Regexp* parent_arg); + + private: + // These functions are declared inside SimplifyWalker so that + // they can edit the private fields of the Regexps they construct. + + // Creates a concatenation of two Regexp, consuming refs to re1 and re2. + // Caller must Decref return value when done with it. + static Regexp* Concat2(Regexp* re1, Regexp* re2, Regexp::ParseFlags flags); + + // Simplifies the expression re{min,max} in terms of *, +, and ?. + // Returns a new regexp. Does not edit re. Does not consume reference to re. + // Caller must Decref return value when done with it. + static Regexp* SimplifyRepeat(Regexp* re, int min, int max, + Regexp::ParseFlags parse_flags); + + // Simplifies a character class by expanding any named classes + // into rune ranges. Does not edit re. Does not consume ref to re. + // Caller must Decref return value when done with it. + static Regexp* SimplifyCharClass(Regexp* re); + + SimplifyWalker(const SimplifyWalker&) = delete; + SimplifyWalker& operator=(const SimplifyWalker&) = delete; +}; + +// Simplifies a regular expression, returning a new regexp. +// The new regexp uses traditional Unix egrep features only, +// plus the Perl (?:) non-capturing parentheses. +// Otherwise, no POSIX or Perl additions. The new regexp +// captures exactly the same subexpressions (with the same indices) +// as the original. +// Does not edit current object. +// Caller must Decref() return value when done with it. + +Regexp* Regexp::Simplify() { + CoalesceWalker cw; + Regexp* cre = cw.Walk(this, NULL); + if (cre == NULL) + return cre; + SimplifyWalker sw; + Regexp* sre = sw.Walk(cre, NULL); + cre->Decref(); + return sre; +} + +#define Simplify DontCallSimplify // Avoid accidental recursion + +// Utility function for PostVisit implementations that compares re->sub() with +// child_args to determine whether any child_args changed. In the common case, +// where nothing changed, calls Decref() for all child_args and returns false, +// so PostVisit must return re->Incref(). Otherwise, returns true. +static bool ChildArgsChanged(Regexp* re, Regexp** child_args) { + for (int i = 0; i < re->nsub(); i++) { + Regexp* sub = re->sub()[i]; + Regexp* newsub = child_args[i]; + if (newsub != sub) + return true; + } + for (int i = 0; i < re->nsub(); i++) { + Regexp* newsub = child_args[i]; + newsub->Decref(); + } + return false; +} + +Regexp* CoalesceWalker::Copy(Regexp* re) { + return re->Incref(); +} + +Regexp* CoalesceWalker::ShortVisit(Regexp* re, Regexp* parent_arg) { + // This should never be called, since we use Walk and not + // WalkExponential. + LOG(DFATAL) << "CoalesceWalker::ShortVisit called"; + return re->Incref(); +} + +Regexp* CoalesceWalker::PostVisit(Regexp* re, + Regexp* parent_arg, + Regexp* pre_arg, + Regexp** child_args, + int nchild_args) { + if (re->nsub() == 0) + return re->Incref(); + + if (re->op() != kRegexpConcat) { + if (!ChildArgsChanged(re, child_args)) + return re->Incref(); + + // Something changed. Build a new op. + Regexp* nre = new Regexp(re->op(), re->parse_flags()); + nre->AllocSub(re->nsub()); + Regexp** nre_subs = nre->sub(); + for (int i = 0; i < re->nsub(); i++) + nre_subs[i] = child_args[i]; + // Repeats and Captures have additional data that must be copied. + if (re->op() == kRegexpRepeat) { + nre->min_ = re->min(); + nre->max_ = re->max(); + } else if (re->op() == kRegexpCapture) { + nre->cap_ = re->cap(); + } + return nre; + } + + bool can_coalesce = false; + for (int i = 0; i < re->nsub(); i++) { + if (i+1 < re->nsub() && + CanCoalesce(child_args[i], child_args[i+1])) { + can_coalesce = true; + break; + } + } + if (!can_coalesce) { + if (!ChildArgsChanged(re, child_args)) + return re->Incref(); + + // Something changed. Build a new op. + Regexp* nre = new Regexp(re->op(), re->parse_flags()); + nre->AllocSub(re->nsub()); + Regexp** nre_subs = nre->sub(); + for (int i = 0; i < re->nsub(); i++) + nre_subs[i] = child_args[i]; + return nre; + } + + for (int i = 0; i < re->nsub(); i++) { + if (i+1 < re->nsub() && + CanCoalesce(child_args[i], child_args[i+1])) + DoCoalesce(&child_args[i], &child_args[i+1]); + } + // Determine how many empty matches were left by DoCoalesce. + int n = 0; + for (int i = n; i < re->nsub(); i++) { + if (child_args[i]->op() == kRegexpEmptyMatch) + n++; + } + // Build a new op. + Regexp* nre = new Regexp(re->op(), re->parse_flags()); + nre->AllocSub(re->nsub() - n); + Regexp** nre_subs = nre->sub(); + for (int i = 0, j = 0; i < re->nsub(); i++) { + if (child_args[i]->op() == kRegexpEmptyMatch) { + child_args[i]->Decref(); + continue; + } + nre_subs[j] = child_args[i]; + j++; + } + return nre; +} + +bool CoalesceWalker::CanCoalesce(Regexp* r1, Regexp* r2) { + // r1 must be a star/plus/quest/repeat of a literal, char class, any char or + // any byte. + if ((r1->op() == kRegexpStar || + r1->op() == kRegexpPlus || + r1->op() == kRegexpQuest || + r1->op() == kRegexpRepeat) && + (r1->sub()[0]->op() == kRegexpLiteral || + r1->sub()[0]->op() == kRegexpCharClass || + r1->sub()[0]->op() == kRegexpAnyChar || + r1->sub()[0]->op() == kRegexpAnyByte)) { + // r2 must be a star/plus/quest/repeat of the same literal, char class, + // any char or any byte. + if ((r2->op() == kRegexpStar || + r2->op() == kRegexpPlus || + r2->op() == kRegexpQuest || + r2->op() == kRegexpRepeat) && + Regexp::Equal(r1->sub()[0], r2->sub()[0]) && + // The parse flags must be consistent. + ((r1->parse_flags() & Regexp::NonGreedy) == + (r2->parse_flags() & Regexp::NonGreedy))) { + return true; + } + // ... OR an occurrence of that literal, char class, any char or any byte + if (Regexp::Equal(r1->sub()[0], r2)) { + return true; + } + // ... OR a literal string that begins with that literal. + if (r1->sub()[0]->op() == kRegexpLiteral && + r2->op() == kRegexpLiteralString && + r2->runes()[0] == r1->sub()[0]->rune() && + // The parse flags must be consistent. + ((r1->sub()[0]->parse_flags() & Regexp::FoldCase) == + (r2->parse_flags() & Regexp::FoldCase))) { + return true; + } + } + return false; +} + +void CoalesceWalker::DoCoalesce(Regexp** r1ptr, Regexp** r2ptr) { + Regexp* r1 = *r1ptr; + Regexp* r2 = *r2ptr; + + Regexp* nre = Regexp::Repeat( + r1->sub()[0]->Incref(), r1->parse_flags(), 0, 0); + + switch (r1->op()) { + case kRegexpStar: + nre->min_ = 0; + nre->max_ = -1; + break; + + case kRegexpPlus: + nre->min_ = 1; + nre->max_ = -1; + break; + + case kRegexpQuest: + nre->min_ = 0; + nre->max_ = 1; + break; + + case kRegexpRepeat: + nre->min_ = r1->min(); + nre->max_ = r1->max(); + break; + + default: + LOG(DFATAL) << "DoCoalesce failed: r1->op() is " << r1->op(); + nre->Decref(); + return; + } + + switch (r2->op()) { + case kRegexpStar: + nre->max_ = -1; + goto LeaveEmpty; + + case kRegexpPlus: + nre->min_++; + nre->max_ = -1; + goto LeaveEmpty; + + case kRegexpQuest: + if (nre->max() != -1) + nre->max_++; + goto LeaveEmpty; + + case kRegexpRepeat: + nre->min_ += r2->min(); + if (r2->max() == -1) + nre->max_ = -1; + else if (nre->max() != -1) + nre->max_ += r2->max(); + goto LeaveEmpty; + + case kRegexpLiteral: + case kRegexpCharClass: + case kRegexpAnyChar: + case kRegexpAnyByte: + nre->min_++; + if (nre->max() != -1) + nre->max_++; + goto LeaveEmpty; + + LeaveEmpty: + *r1ptr = new Regexp(kRegexpEmptyMatch, Regexp::NoParseFlags); + *r2ptr = nre; + break; + + case kRegexpLiteralString: { + Rune r = r1->sub()[0]->rune(); + // Determine how much of the literal string is removed. + // We know that we have at least one rune. :) + int n = 1; + while (n < r2->nrunes() && r2->runes()[n] == r) + n++; + nre->min_ += n; + if (nre->max() != -1) + nre->max_ += n; + if (n == r2->nrunes()) + goto LeaveEmpty; + *r1ptr = nre; + *r2ptr = Regexp::LiteralString( + &r2->runes()[n], r2->nrunes() - n, r2->parse_flags()); + break; + } + + default: + LOG(DFATAL) << "DoCoalesce failed: r2->op() is " << r2->op(); + nre->Decref(); + return; + } + + r1->Decref(); + r2->Decref(); +} + +Regexp* SimplifyWalker::Copy(Regexp* re) { + return re->Incref(); +} + +Regexp* SimplifyWalker::ShortVisit(Regexp* re, Regexp* parent_arg) { + // This should never be called, since we use Walk and not + // WalkExponential. + LOG(DFATAL) << "SimplifyWalker::ShortVisit called"; + return re->Incref(); +} + +Regexp* SimplifyWalker::PreVisit(Regexp* re, Regexp* parent_arg, bool* stop) { + if (re->simple()) { + *stop = true; + return re->Incref(); + } + return NULL; +} + +Regexp* SimplifyWalker::PostVisit(Regexp* re, + Regexp* parent_arg, + Regexp* pre_arg, + Regexp** child_args, + int nchild_args) { + switch (re->op()) { + case kRegexpNoMatch: + case kRegexpEmptyMatch: + case kRegexpLiteral: + case kRegexpLiteralString: + case kRegexpBeginLine: + case kRegexpEndLine: + case kRegexpBeginText: + case kRegexpWordBoundary: + case kRegexpNoWordBoundary: + case kRegexpEndText: + case kRegexpAnyChar: + case kRegexpAnyByte: + case kRegexpHaveMatch: + // All these are always simple. + re->simple_ = true; + return re->Incref(); + + case kRegexpConcat: + case kRegexpAlternate: { + // These are simple as long as the subpieces are simple. + if (!ChildArgsChanged(re, child_args)) { + re->simple_ = true; + return re->Incref(); + } + Regexp* nre = new Regexp(re->op(), re->parse_flags()); + nre->AllocSub(re->nsub()); + Regexp** nre_subs = nre->sub(); + for (int i = 0; i < re->nsub(); i++) + nre_subs[i] = child_args[i]; + nre->simple_ = true; + return nre; + } + + case kRegexpCapture: { + Regexp* newsub = child_args[0]; + if (newsub == re->sub()[0]) { + newsub->Decref(); + re->simple_ = true; + return re->Incref(); + } + Regexp* nre = new Regexp(kRegexpCapture, re->parse_flags()); + nre->AllocSub(1); + nre->sub()[0] = newsub; + nre->cap_ = re->cap(); + nre->simple_ = true; + return nre; + } + + case kRegexpStar: + case kRegexpPlus: + case kRegexpQuest: { + Regexp* newsub = child_args[0]; + // Special case: repeat the empty string as much as + // you want, but it's still the empty string. + if (newsub->op() == kRegexpEmptyMatch) + return newsub; + + // These are simple as long as the subpiece is simple. + if (newsub == re->sub()[0]) { + newsub->Decref(); + re->simple_ = true; + return re->Incref(); + } + + // These are also idempotent if flags are constant. + if (re->op() == newsub->op() && + re->parse_flags() == newsub->parse_flags()) + return newsub; + + Regexp* nre = new Regexp(re->op(), re->parse_flags()); + nre->AllocSub(1); + nre->sub()[0] = newsub; + nre->simple_ = true; + return nre; + } + + case kRegexpRepeat: { + Regexp* newsub = child_args[0]; + // Special case: repeat the empty string as much as + // you want, but it's still the empty string. + if (newsub->op() == kRegexpEmptyMatch) + return newsub; + + Regexp* nre = SimplifyRepeat(newsub, re->min_, re->max_, + re->parse_flags()); + newsub->Decref(); + nre->simple_ = true; + return nre; + } + + case kRegexpCharClass: { + Regexp* nre = SimplifyCharClass(re); + nre->simple_ = true; + return nre; + } + } + + LOG(ERROR) << "Simplify case not handled: " << re->op(); + return re->Incref(); +} + +// Creates a concatenation of two Regexp, consuming refs to re1 and re2. +// Returns a new Regexp, handing the ref to the caller. +Regexp* SimplifyWalker::Concat2(Regexp* re1, Regexp* re2, + Regexp::ParseFlags parse_flags) { + Regexp* re = new Regexp(kRegexpConcat, parse_flags); + re->AllocSub(2); + Regexp** subs = re->sub(); + subs[0] = re1; + subs[1] = re2; + return re; +} + +// Simplifies the expression re{min,max} in terms of *, +, and ?. +// Returns a new regexp. Does not edit re. Does not consume reference to re. +// Caller must Decref return value when done with it. +// The result will *not* necessarily have the right capturing parens +// if you call ToString() and re-parse it: (x){2} becomes (x)(x), +// but in the Regexp* representation, both (x) are marked as $1. +Regexp* SimplifyWalker::SimplifyRepeat(Regexp* re, int min, int max, + Regexp::ParseFlags f) { + // x{n,} means at least n matches of x. + if (max == -1) { + // Special case: x{0,} is x* + if (min == 0) + return Regexp::Star(re->Incref(), f); + + // Special case: x{1,} is x+ + if (min == 1) + return Regexp::Plus(re->Incref(), f); + + // General case: x{4,} is xxxx+ + PODArray nre_subs(min); + for (int i = 0; i < min-1; i++) + nre_subs[i] = re->Incref(); + nre_subs[min-1] = Regexp::Plus(re->Incref(), f); + return Regexp::Concat(nre_subs.data(), min, f); + } + + // Special case: (x){0} matches only empty string. + if (min == 0 && max == 0) + return new Regexp(kRegexpEmptyMatch, f); + + // Special case: x{1} is just x. + if (min == 1 && max == 1) + return re->Incref(); + + // General case: x{n,m} means n copies of x and m copies of x?. + // The machine will do less work if we nest the final m copies, + // so that x{2,5} = xx(x(x(x)?)?)? + + // Build leading prefix: xx. Capturing only on the last one. + Regexp* nre = NULL; + if (min > 0) { + PODArray nre_subs(min); + for (int i = 0; i < min; i++) + nre_subs[i] = re->Incref(); + nre = Regexp::Concat(nre_subs.data(), min, f); + } + + // Build and attach suffix: (x(x(x)?)?)? + if (max > min) { + Regexp* suf = Regexp::Quest(re->Incref(), f); + for (int i = min+1; i < max; i++) + suf = Regexp::Quest(Concat2(re->Incref(), suf, f), f); + if (nre == NULL) + nre = suf; + else + nre = Concat2(nre, suf, f); + } + + if (nre == NULL) { + // Some degenerate case, like min > max, or min < max < 0. + // This shouldn't happen, because the parser rejects such regexps. + LOG(DFATAL) << "Malformed repeat " << re->ToString() << " " << min << " " << max; + return new Regexp(kRegexpNoMatch, f); + } + + return nre; +} + +// Simplifies a character class. +// Caller must Decref return value when done with it. +Regexp* SimplifyWalker::SimplifyCharClass(Regexp* re) { + CharClass* cc = re->cc(); + + // Special cases + if (cc->empty()) + return new Regexp(kRegexpNoMatch, re->parse_flags()); + if (cc->full()) + return new Regexp(kRegexpAnyChar, re->parse_flags()); + + return re->Incref(); +} + +} // namespace re2 diff --git a/extern/re2/re2/stringpiece.cc b/extern/re2/re2/stringpiece.cc new file mode 100644 index 0000000000..ef2e2874ea --- /dev/null +++ b/extern/re2/re2/stringpiece.cc @@ -0,0 +1,65 @@ +// Copyright 2004 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "re2/stringpiece.h" + +#include + +#include "util/util.h" + +namespace re2 { + +const StringPiece::size_type StringPiece::npos; // initialized in stringpiece.h + +StringPiece::size_type StringPiece::copy(char* buf, size_type n, + size_type pos) const { + size_type ret = std::min(size_ - pos, n); + memcpy(buf, data_ + pos, ret); + return ret; +} + +StringPiece StringPiece::substr(size_type pos, size_type n) const { + if (pos > size_) pos = size_; + if (n > size_ - pos) n = size_ - pos; + return StringPiece(data_ + pos, n); +} + +StringPiece::size_type StringPiece::find(const StringPiece& s, + size_type pos) const { + if (pos > size_) return npos; + const_pointer result = std::search(data_ + pos, data_ + size_, + s.data_, s.data_ + s.size_); + size_type xpos = result - data_; + return xpos + s.size_ <= size_ ? xpos : npos; +} + +StringPiece::size_type StringPiece::find(char c, size_type pos) const { + if (size_ <= 0 || pos >= size_) return npos; + const_pointer result = std::find(data_ + pos, data_ + size_, c); + return result != data_ + size_ ? result - data_ : npos; +} + +StringPiece::size_type StringPiece::rfind(const StringPiece& s, + size_type pos) const { + if (size_ < s.size_) return npos; + if (s.size_ == 0) return std::min(size_, pos); + const_pointer last = data_ + std::min(size_ - s.size_, pos) + s.size_; + const_pointer result = std::find_end(data_, last, s.data_, s.data_ + s.size_); + return result != last ? result - data_ : npos; +} + +StringPiece::size_type StringPiece::rfind(char c, size_type pos) const { + if (size_ <= 0) return npos; + for (size_t i = std::min(pos + 1, size_); i != 0;) { + if (data_[--i] == c) return i; + } + return npos; +} + +std::ostream& operator<<(std::ostream& o, const StringPiece& p) { + o.write(p.data(), p.size()); + return o; +} + +} // namespace re2 diff --git a/extern/re2/re2/stringpiece.h b/extern/re2/re2/stringpiece.h new file mode 100644 index 0000000000..1d9c2d3d2c --- /dev/null +++ b/extern/re2/re2/stringpiece.h @@ -0,0 +1,210 @@ +// Copyright 2001-2010 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_STRINGPIECE_H_ +#define RE2_STRINGPIECE_H_ + +// A string-like object that points to a sized piece of memory. +// +// Functions or methods may use const StringPiece& parameters to accept either +// a "const char*" or a "string" value that will be implicitly converted to +// a StringPiece. The implicit conversion means that it is often appropriate +// to include this .h file in other files rather than forward-declaring +// StringPiece as would be appropriate for most other Google classes. +// +// Systematic usage of StringPiece is encouraged as it will reduce unnecessary +// conversions from "const char*" to "string" and back again. +// +// +// Arghh! I wish C++ literals were "string". + +// Doing this simplifies the logic below. +#ifndef __has_include +#define __has_include(x) 0 +#endif + +#include +#include +#include +#include +#include +#include +#if __has_include() && __cplusplus >= 201703L +#include +#endif + +namespace re2 { + +class StringPiece { + public: + typedef std::char_traits traits_type; + typedef char value_type; + typedef char* pointer; + typedef const char* const_pointer; + typedef char& reference; + typedef const char& const_reference; + typedef const char* const_iterator; + typedef const_iterator iterator; + typedef std::reverse_iterator const_reverse_iterator; + typedef const_reverse_iterator reverse_iterator; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + static const size_type npos = static_cast(-1); + + // We provide non-explicit singleton constructors so users can pass + // in a "const char*" or a "string" wherever a "StringPiece" is + // expected. + StringPiece() + : data_(NULL), size_(0) {} +#if __has_include() && __cplusplus >= 201703L + StringPiece(const std::string_view& str) + : data_(str.data()), size_(str.size()) {} +#endif + StringPiece(const std::string& str) + : data_(str.data()), size_(str.size()) {} + StringPiece(const char* str) + : data_(str), size_(str == NULL ? 0 : strlen(str)) {} + StringPiece(const char* str, size_type len) + : data_(str), size_(len) {} + + const_iterator begin() const { return data_; } + const_iterator end() const { return data_ + size_; } + const_reverse_iterator rbegin() const { + return const_reverse_iterator(data_ + size_); + } + const_reverse_iterator rend() const { + return const_reverse_iterator(data_); + } + + size_type size() const { return size_; } + size_type length() const { return size_; } + bool empty() const { return size_ == 0; } + + const_reference operator[](size_type i) const { return data_[i]; } + const_pointer data() const { return data_; } + + void remove_prefix(size_type n) { + data_ += n; + size_ -= n; + } + + void remove_suffix(size_type n) { + size_ -= n; + } + + void set(const char* str) { + data_ = str; + size_ = str == NULL ? 0 : strlen(str); + } + + void set(const char* str, size_type len) { + data_ = str; + size_ = len; + } + + // Converts to `std::basic_string`. + template + explicit operator std::basic_string() const { + if (!data_) return {}; + return std::basic_string(data_, size_); + } + + std::string as_string() const { + return std::string(data_, size_); + } + + // We also define ToString() here, since many other string-like + // interfaces name the routine that converts to a C++ string + // "ToString", and it's confusing to have the method that does that + // for a StringPiece be called "as_string()". We also leave the + // "as_string()" method defined here for existing code. + std::string ToString() const { + return std::string(data_, size_); + } + + void CopyToString(std::string* target) const { + target->assign(data_, size_); + } + + void AppendToString(std::string* target) const { + target->append(data_, size_); + } + + size_type copy(char* buf, size_type n, size_type pos = 0) const; + StringPiece substr(size_type pos = 0, size_type n = npos) const; + + int compare(const StringPiece& x) const { + size_type min_size = std::min(size(), x.size()); + if (min_size > 0) { + int r = memcmp(data(), x.data(), min_size); + if (r < 0) return -1; + if (r > 0) return 1; + } + if (size() < x.size()) return -1; + if (size() > x.size()) return 1; + return 0; + } + + // Does "this" start with "x"? + bool starts_with(const StringPiece& x) const { + return x.empty() || + (size() >= x.size() && memcmp(data(), x.data(), x.size()) == 0); + } + + // Does "this" end with "x"? + bool ends_with(const StringPiece& x) const { + return x.empty() || + (size() >= x.size() && + memcmp(data() + (size() - x.size()), x.data(), x.size()) == 0); + } + + bool contains(const StringPiece& s) const { + return find(s) != npos; + } + + size_type find(const StringPiece& s, size_type pos = 0) const; + size_type find(char c, size_type pos = 0) const; + size_type rfind(const StringPiece& s, size_type pos = npos) const; + size_type rfind(char c, size_type pos = npos) const; + + private: + const_pointer data_; + size_type size_; +}; + +inline bool operator==(const StringPiece& x, const StringPiece& y) { + StringPiece::size_type len = x.size(); + if (len != y.size()) return false; + return x.data() == y.data() || len == 0 || + memcmp(x.data(), y.data(), len) == 0; +} + +inline bool operator!=(const StringPiece& x, const StringPiece& y) { + return !(x == y); +} + +inline bool operator<(const StringPiece& x, const StringPiece& y) { + StringPiece::size_type min_size = std::min(x.size(), y.size()); + int r = min_size == 0 ? 0 : memcmp(x.data(), y.data(), min_size); + return (r < 0) || (r == 0 && x.size() < y.size()); +} + +inline bool operator>(const StringPiece& x, const StringPiece& y) { + return y < x; +} + +inline bool operator<=(const StringPiece& x, const StringPiece& y) { + return !(x > y); +} + +inline bool operator>=(const StringPiece& x, const StringPiece& y) { + return !(x < y); +} + +// Allow StringPiece to be logged. +std::ostream& operator<<(std::ostream& o, const StringPiece& p); + +} // namespace re2 + +#endif // RE2_STRINGPIECE_H_ diff --git a/extern/re2/re2/testing/backtrack.cc b/extern/re2/re2/testing/backtrack.cc new file mode 100644 index 0000000000..ae9fd82bc7 --- /dev/null +++ b/extern/re2/re2/testing/backtrack.cc @@ -0,0 +1,273 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tested by search_test.cc, exhaustive_test.cc, tester.cc +// +// Prog::UnsafeSearchBacktrack is a backtracking regular expression search, +// except that it remembers where it has been, trading a lot of +// memory for a lot of time. It exists only for testing purposes. +// +// Let me repeat that. +// +// THIS CODE SHOULD NEVER BE USED IN PRODUCTION: +// - It uses a ton of memory. +// - It uses a ton of stack. +// - It uses CHECK and LOG(FATAL). +// - It implements unanchored search by repeated anchored search. +// +// On the other hand, it is very simple and a good reference +// implementation for the more complicated regexp packages. +// +// In BUILD, this file is linked into the ":testing" library, +// not the main library, in order to make it harder to pick up +// accidentally. + +#include +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "re2/prog.h" +#include "re2/regexp.h" + +namespace re2 { + +// Backtracker holds the state for a backtracking search. +// +// Excluding the search parameters, the main search state +// is just the "capture registers", which record, for the +// current execution, the string position at which each +// parenthesis was passed. cap_[0] and cap_[1] are the +// left and right parenthesis in $0, cap_[2] and cap_[3] in $1, etc. +// +// To avoid infinite loops during backtracking on expressions +// like (a*)*, the visited_[] bitmap marks the (state, string-position) +// pairs that have already been explored and are thus not worth +// re-exploring if we get there via another path. Modern backtracking +// libraries engineer their program representation differently, to make +// such infinite loops possible to avoid without keeping a giant visited_ +// bitmap, but visited_ works fine for a reference implementation +// and it has the nice benefit of making the search run in linear time. +class Backtracker { + public: + explicit Backtracker(Prog* prog); + ~Backtracker(); + + bool Search(const StringPiece& text, const StringPiece& context, + bool anchored, bool longest, + StringPiece* submatch, int nsubmatch); + + private: + // Explores from instruction id at string position p looking for a match. + // Returns true if found (so that caller can stop trying other possibilities). + bool Visit(int id, const char* p); + + // Tries instruction id at string position p. + // Returns true if a match is found. + bool Try(int id, const char* p); + + // Search parameters + Prog* prog_; // program being run + StringPiece text_; // text being searched + StringPiece context_; // greater context of text being searched + bool anchored_; // whether search is anchored at text.begin() + bool longest_; // whether search wants leftmost-longest match + bool endmatch_; // whether search must end at text.end() + StringPiece *submatch_; // submatches to fill in + int nsubmatch_; // # of submatches to fill in + + // Search state + const char* cap_[64]; // capture registers + uint32_t *visited_; // bitmap: (Inst*, char*) pairs already backtracked + size_t nvisited_; // # of words in bitmap +}; + +Backtracker::Backtracker(Prog* prog) + : prog_(prog), + anchored_(false), + longest_(false), + endmatch_(false), + submatch_(NULL), + nsubmatch_(0), + visited_(NULL), + nvisited_(0) { +} + +Backtracker::~Backtracker() { + delete[] visited_; +} + +// Runs a backtracking search. +bool Backtracker::Search(const StringPiece& text, const StringPiece& context, + bool anchored, bool longest, + StringPiece* submatch, int nsubmatch) { + text_ = text; + context_ = context; + if (context_.begin() == NULL) + context_ = text; + if (prog_->anchor_start() && text.begin() > context_.begin()) + return false; + if (prog_->anchor_end() && text.end() < context_.end()) + return false; + anchored_ = anchored | prog_->anchor_start(); + longest_ = longest | prog_->anchor_end(); + endmatch_ = prog_->anchor_end(); + submatch_ = submatch; + nsubmatch_ = nsubmatch; + CHECK_LT(2*nsubmatch_, static_cast(arraysize(cap_))); + memset(cap_, 0, sizeof cap_); + + // We use submatch_[0] for our own bookkeeping, + // so it had better exist. + StringPiece sp0; + if (nsubmatch < 1) { + submatch_ = &sp0; + nsubmatch_ = 1; + } + submatch_[0] = StringPiece(); + + // Allocate new visited_ bitmap -- size is proportional + // to text, so have to reallocate on each call to Search. + delete[] visited_; + nvisited_ = (prog_->size()*(text.size()+1) + 31)/32; + visited_ = new uint32_t[nvisited_]; + memset(visited_, 0, nvisited_*sizeof visited_[0]); + + // Anchored search must start at text.begin(). + if (anchored_) { + cap_[0] = text.begin(); + return Visit(prog_->start(), text.begin()); + } + + // Unanchored search, starting from each possible text position. + // Notice that we have to try the empty string at the end of + // the text, so the loop condition is p <= text.end(), not p < text.end(). + for (const char* p = text.begin(); p <= text.end(); p++) { + cap_[0] = p; + if (Visit(prog_->start(), p)) // Match must be leftmost; done. + return true; + } + return false; +} + +// Explores from instruction id at string position p looking for a match. +// Return true if found (so that caller can stop trying other possibilities). +bool Backtracker::Visit(int id, const char* p) { + // Check bitmap. If we've already explored from here, + // either it didn't match or it did but we're hoping for a better match. + // Either way, don't go down that road again. + CHECK(p <= text_.end()); + size_t n = id*(text_.size()+1) + (p - text_.begin()); + CHECK_LT(n/32, nvisited_); + if (visited_[n/32] & (1 << (n&31))) + return false; + visited_[n/32] |= 1 << (n&31); + + Prog::Inst* ip = prog_->inst(id); + if (Try(id, p)) { + if (longest_ && !ip->last()) + Visit(id+1, p); + return true; + } + if (!ip->last()) + return Visit(id+1, p); + return false; +} + +// Tries instruction id at string position p. +// Returns true if a match is found. +bool Backtracker::Try(int id, const char* p) { + // Pick out byte at current position. If at end of string, + // have to explore in hope of finishing a match. Use impossible byte -1. + int c = -1; + if (p < text_.end()) + c = *p & 0xFF; + + Prog::Inst* ip = prog_->inst(id); + switch (ip->opcode()) { + default: + LOG(FATAL) << "Unexpected opcode: " << (int)ip->opcode(); + return false; // not reached + + case kInstAltMatch: + // Ignored. + return false; + + case kInstByteRange: + if (ip->Matches(c)) + return Visit(ip->out(), p+1); + return false; + + case kInstCapture: + if (0 <= ip->cap() && + ip->cap() < static_cast(arraysize(cap_))) { + // Capture p to register, but save old value. + const char* q = cap_[ip->cap()]; + cap_[ip->cap()] = p; + bool ret = Visit(ip->out(), p); + // Restore old value as we backtrack. + cap_[ip->cap()] = q; + return ret; + } + return Visit(ip->out(), p); + + case kInstEmptyWidth: + if (ip->empty() & ~Prog::EmptyFlags(context_, p)) + return false; + return Visit(ip->out(), p); + + case kInstNop: + return Visit(ip->out(), p); + + case kInstMatch: + // We found a match. If it's the best so far, record the + // parameters in the caller's submatch_ array. + if (endmatch_ && p != context_.end()) + return false; + cap_[1] = p; + if (submatch_[0].data() == NULL || // First match so far ... + (longest_ && p > submatch_[0].end())) { // ... or better match + for (int i = 0; i < nsubmatch_; i++) + submatch_[i] = StringPiece( + cap_[2 * i], static_cast(cap_[2 * i + 1] - cap_[2 * i])); + } + return true; + + case kInstFail: + return false; + } +} + +// Runs a backtracking search. +bool Prog::UnsafeSearchBacktrack(const StringPiece& text, + const StringPiece& context, + Anchor anchor, + MatchKind kind, + StringPiece* match, + int nmatch) { + // If full match, we ask for an anchored longest match + // and then check that match[0] == text. + // So make sure match[0] exists. + StringPiece sp0; + if (kind == kFullMatch) { + anchor = kAnchored; + if (nmatch < 1) { + match = &sp0; + nmatch = 1; + } + } + + // Run the search. + Backtracker b(this); + bool anchored = anchor == kAnchored; + bool longest = kind != kFirstMatch; + if (!b.Search(text, context, anchored, longest, match, nmatch)) + return false; + if (kind == kFullMatch && match[0].end() != text.end()) + return false; + return true; +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/charclass_test.cc b/extern/re2/re2/testing/charclass_test.cc new file mode 100644 index 0000000000..a2837a69a3 --- /dev/null +++ b/extern/re2/re2/testing/charclass_test.cc @@ -0,0 +1,226 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test character class manipulations. + +#include + +#include "util/test.h" +#include "util/utf.h" +#include "re2/regexp.h" + +namespace re2 { + +struct CCTest { + struct { + Rune lo; + Rune hi; + } add[10]; + int remove; + struct { + Rune lo; + Rune hi; + } final[10]; +}; + +static CCTest tests[] = { + { { { 10, 20 }, {-1} }, -1, + { { 10, 20 }, {-1} } }, + + { { { 10, 20 }, { 20, 30 }, {-1} }, -1, + { { 10, 30 }, {-1} } }, + + { { { 10, 20 }, { 30, 40 }, { 20, 30 }, {-1} }, -1, + { { 10, 40 }, {-1} } }, + + { { { 0, 50 }, { 20, 30 }, {-1} }, -1, + { { 0, 50 }, {-1} } }, + + { { { 10, 11 }, { 13, 14 }, { 16, 17 }, { 19, 20 }, { 22, 23 }, {-1} }, -1, + { { 10, 11 }, { 13, 14 }, { 16, 17 }, { 19, 20 }, { 22, 23 }, {-1} } }, + + { { { 13, 14 }, { 10, 11 }, { 22, 23 }, { 19, 20 }, { 16, 17 }, {-1} }, -1, + { { 10, 11 }, { 13, 14 }, { 16, 17 }, { 19, 20 }, { 22, 23 }, {-1} } }, + + { { { 13, 14 }, { 10, 11 }, { 22, 23 }, { 19, 20 }, { 16, 17 }, {-1} }, -1, + { { 10, 11 }, { 13, 14 }, { 16, 17 }, { 19, 20 }, { 22, 23 }, {-1} } }, + + { { { 13, 14 }, { 10, 11 }, { 22, 23 }, { 19, 20 }, { 16, 17 }, { 5, 25 }, {-1} }, -1, + { { 5, 25 }, {-1} } }, + + { { { 13, 14 }, { 10, 11 }, { 22, 23 }, { 19, 20 }, { 16, 17 }, { 12, 21 }, {-1} }, -1, + { { 10, 23 }, {-1} } }, + + // These check boundary cases during negation. + { { { 0, Runemax }, {-1} }, -1, + { { 0, Runemax }, {-1} } }, + + { { { 0, 50 }, {-1} }, -1, + { { 0, 50 }, {-1} } }, + + { { { 50, Runemax }, {-1} }, -1, + { { 50, Runemax }, {-1} } }, + + // Check RemoveAbove. + { { { 50, Runemax }, {-1} }, 255, + { { 50, 255 }, {-1} } }, + + { { { 50, Runemax }, {-1} }, 65535, + { { 50, 65535 }, {-1} } }, + + { { { 50, Runemax }, {-1} }, Runemax, + { { 50, Runemax }, {-1} } }, + + { { { 50, 60 }, { 250, 260 }, { 350, 360 }, {-1} }, 255, + { { 50, 60 }, { 250, 255 }, {-1} } }, + + { { { 50, 60 }, {-1} }, 255, + { { 50, 60 }, {-1} } }, + + { { { 350, 360 }, {-1} }, 255, + { {-1} } }, + + { { {-1} }, 255, + { {-1} } }, +}; + +template +static void Broke(const char *desc, const CCTest* t, CharClass* cc) { + if (t == NULL) { + printf("\t%s:", desc); + } else { + printf("\n"); + printf("CharClass added: [%s]", desc); + for (int k = 0; t->add[k].lo >= 0; k++) + printf(" %d-%d", t->add[k].lo, t->add[k].hi); + printf("\n"); + if (t->remove >= 0) + printf("Removed > %d\n", t->remove); + printf("\twant:"); + for (int k = 0; t->final[k].lo >= 0; k++) + printf(" %d-%d", t->final[k].lo, t->final[k].hi); + printf("\n"); + printf("\thave:"); + } + + for (typename CharClass::iterator it = cc->begin(); it != cc->end(); ++it) + printf(" %d-%d", it->lo, it->hi); + printf("\n"); +} + +bool ShouldContain(CCTest *t, int x) { + for (int j = 0; t->final[j].lo >= 0; j++) + if (t->final[j].lo <= x && x <= t->final[j].hi) + return true; + return false; +} + +// Helpers to make templated CorrectCC work with both CharClass and CharClassBuilder. + +CharClass* Negate(CharClass *cc) { + return cc->Negate(); +} + +void Delete(CharClass* cc) { + cc->Delete(); +} + +CharClassBuilder* Negate(CharClassBuilder* cc) { + CharClassBuilder* ncc = cc->Copy(); + ncc->Negate(); + return ncc; +} + +void Delete(CharClassBuilder* cc) { + delete cc; +} + +template +bool CorrectCC(CharClass *cc, CCTest *t, const char *desc) { + typename CharClass::iterator it = cc->begin(); + int size = 0; + for (int j = 0; t->final[j].lo >= 0; j++, ++it) { + if (it == cc->end() || + it->lo != t->final[j].lo || + it->hi != t->final[j].hi) { + Broke(desc, t, cc); + return false; + } + size += it->hi - it->lo + 1; + } + if (it != cc->end()) { + Broke(desc, t, cc); + return false; + } + if (cc->size() != size) { + Broke(desc, t, cc); + printf("wrong size: want %d have %d\n", size, cc->size()); + return false; + } + + for (int j = 0; j < 101; j++) { + if (j == 100) + j = Runemax; + if (ShouldContain(t, j) != cc->Contains(j)) { + Broke(desc, t, cc); + printf("want contains(%d)=%d, got %d\n", + j, ShouldContain(t, j), cc->Contains(j)); + return false; + } + } + + CharClass* ncc = Negate(cc); + for (int j = 0; j < 101; j++) { + if (j == 100) + j = Runemax; + if (ShouldContain(t, j) == ncc->Contains(j)) { + Broke(desc, t, cc); + Broke("ncc", NULL, ncc); + printf("want ncc contains(%d)!=%d, got %d\n", + j, ShouldContain(t, j), ncc->Contains(j)); + Delete(ncc); + return false; + } + if (ncc->size() != Runemax+1 - cc->size()) { + Broke(desc, t, cc); + Broke("ncc", NULL, ncc); + printf("ncc size should be %d is %d\n", + Runemax+1 - cc->size(), ncc->size()); + Delete(ncc); + return false; + } + } + Delete(ncc); + return true; +} + +TEST(TestCharClassBuilder, Adds) { + int nfail = 0; + for (size_t i = 0; i < arraysize(tests); i++) { + CharClassBuilder ccb; + CCTest* t = &tests[i]; + for (int j = 0; t->add[j].lo >= 0; j++) + ccb.AddRange(t->add[j].lo, t->add[j].hi); + if (t->remove >= 0) + ccb.RemoveAbove(t->remove); + if (!CorrectCC(&ccb, t, "before copy (CharClassBuilder)")) + nfail++; + CharClass* cc = ccb.GetCharClass(); + if (!CorrectCC(cc, t, "before copy (CharClass)")) + nfail++; + cc->Delete(); + + CharClassBuilder *ccb1 = ccb.Copy(); + if (!CorrectCC(ccb1, t, "after copy (CharClassBuilder)")) + nfail++; + cc = ccb.GetCharClass(); + if (!CorrectCC(cc, t, "after copy (CharClass)")) + nfail++; + cc->Delete(); + delete ccb1; + } + EXPECT_EQ(nfail, 0); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/compile_test.cc b/extern/re2/re2/testing/compile_test.cc new file mode 100644 index 0000000000..6b77cf97b9 --- /dev/null +++ b/extern/re2/re2/testing/compile_test.cc @@ -0,0 +1,397 @@ +// Copyright 2007 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test prog.cc, compile.cc + +#include + +#include "util/test.h" +#include "util/logging.h" +#include "re2/regexp.h" +#include "re2/prog.h" + +namespace re2 { + +// Simple input/output tests checking that +// the regexp compiles to the expected code. +// These are just to sanity check the basic implementation. +// The real confidence tests happen by testing the NFA/DFA +// that run the compiled code. + +struct Test { + const char* regexp; + const char* code; +}; + +static Test tests[] = { + { "a", + "3. byte [61-61] 0 -> 4\n" + "4. match! 0\n" }, + { "ab", + "3. byte [61-61] 0 -> 4\n" + "4. byte [62-62] 0 -> 5\n" + "5. match! 0\n" }, + { "a|c", + "3+ byte [61-61] 0 -> 5\n" + "4. byte [63-63] 0 -> 5\n" + "5. match! 0\n" }, + { "a|b", + "3. byte [61-62] 0 -> 4\n" + "4. match! 0\n" }, + { "[ab]", + "3. byte [61-62] 0 -> 4\n" + "4. match! 0\n" }, + { "a+", + "3. byte [61-61] 0 -> 4\n" + "4+ nop -> 3\n" + "5. match! 0\n" }, + { "a+?", + "3. byte [61-61] 0 -> 4\n" + "4+ match! 0\n" + "5. nop -> 3\n" }, + { "a*", + "3+ byte [61-61] 1 -> 3\n" + "4. match! 0\n" }, + { "a*?", + "3+ match! 0\n" + "4. byte [61-61] 0 -> 3\n" }, + { "a?", + "3+ byte [61-61] 1 -> 5\n" + "4. nop -> 5\n" + "5. match! 0\n" }, + { "a??", + "3+ nop -> 5\n" + "4. byte [61-61] 0 -> 5\n" + "5. match! 0\n" }, + { "a{4}", + "3. byte [61-61] 0 -> 4\n" + "4. byte [61-61] 0 -> 5\n" + "5. byte [61-61] 0 -> 6\n" + "6. byte [61-61] 0 -> 7\n" + "7. match! 0\n" }, + { "(a)", + "3. capture 2 -> 4\n" + "4. byte [61-61] 0 -> 5\n" + "5. capture 3 -> 6\n" + "6. match! 0\n" }, + { "(?:a)", + "3. byte [61-61] 0 -> 4\n" + "4. match! 0\n" }, + { "", + "3. match! 0\n" }, + { ".", + "3+ byte [00-09] 0 -> 5\n" + "4. byte [0b-ff] 0 -> 5\n" + "5. match! 0\n" }, + { "[^ab]", + "3+ byte [00-09] 0 -> 6\n" + "4+ byte [0b-60] 0 -> 6\n" + "5. byte [63-ff] 0 -> 6\n" + "6. match! 0\n" }, + { "[Aa]", + "3. byte/i [61-61] 0 -> 4\n" + "4. match! 0\n" }, + { "\\C+", + "3. byte [00-ff] 0 -> 4\n" + "4+ altmatch -> 5 | 6\n" + "5+ nop -> 3\n" + "6. match! 0\n" }, + { "\\C*", + "3+ altmatch -> 4 | 5\n" + "4+ byte [00-ff] 1 -> 3\n" + "5. match! 0\n" }, + { "\\C?", + "3+ byte [00-ff] 1 -> 5\n" + "4. nop -> 5\n" + "5. match! 0\n" }, + // Issue 20992936 + { "[[-`]", + "3. byte [5b-60] 0 -> 4\n" + "4. match! 0\n" }, +}; + +TEST(TestRegexpCompileToProg, Simple) { + int failed = 0; + for (size_t i = 0; i < arraysize(tests); i++) { + const re2::Test& t = tests[i]; + Regexp* re = Regexp::Parse(t.regexp, Regexp::PerlX|Regexp::Latin1, NULL); + if (re == NULL) { + LOG(ERROR) << "Cannot parse: " << t.regexp; + failed++; + continue; + } + Prog* prog = re->CompileToProg(0); + if (prog == NULL) { + LOG(ERROR) << "Cannot compile: " << t.regexp; + re->Decref(); + failed++; + continue; + } + ASSERT_TRUE(re->CompileToProg(1) == NULL); + std::string s = prog->Dump(); + if (s != t.code) { + LOG(ERROR) << "Incorrect compiled code for: " << t.regexp; + LOG(ERROR) << "Want:\n" << t.code; + LOG(ERROR) << "Got:\n" << s; + failed++; + } + delete prog; + re->Decref(); + } + EXPECT_EQ(failed, 0); +} + +static void DumpByteMap(StringPiece pattern, Regexp::ParseFlags flags, + std::string* bytemap) { + Regexp* re = Regexp::Parse(pattern, flags, NULL); + EXPECT_TRUE(re != NULL); + + Prog* prog = re->CompileToProg(0); + EXPECT_TRUE(prog != NULL); + *bytemap = prog->DumpByteMap(); + delete prog; + + re->Decref(); +} + +TEST(TestCompile, Latin1Ranges) { + // The distinct byte ranges involved in the Latin-1 dot ([^\n]). + + std::string bytemap; + + DumpByteMap(".", Regexp::PerlX|Regexp::Latin1, &bytemap); + EXPECT_EQ("[00-09] -> 0\n" + "[0a-0a] -> 1\n" + "[0b-ff] -> 0\n", + bytemap); +} + +TEST(TestCompile, OtherByteMapTests) { + std::string bytemap; + + // Test that "absent" ranges are mapped to the same byte class. + DumpByteMap("[0-9A-Fa-f]+", Regexp::PerlX|Regexp::Latin1, &bytemap); + EXPECT_EQ("[00-2f] -> 0\n" + "[30-39] -> 1\n" + "[3a-40] -> 0\n" + "[41-46] -> 1\n" + "[47-60] -> 0\n" + "[61-66] -> 1\n" + "[67-ff] -> 0\n", + bytemap); + + // Test the byte classes for \b. + DumpByteMap("\\b", Regexp::LikePerl|Regexp::Latin1, &bytemap); + EXPECT_EQ("[00-2f] -> 0\n" + "[30-39] -> 1\n" + "[3a-40] -> 0\n" + "[41-5a] -> 1\n" + "[5b-5e] -> 0\n" + "[5f-5f] -> 1\n" + "[60-60] -> 0\n" + "[61-7a] -> 1\n" + "[7b-ff] -> 0\n", + bytemap); + + // Bug in the ASCII case-folding optimization created too many byte classes. + DumpByteMap("[^_]", Regexp::LikePerl|Regexp::Latin1, &bytemap); + EXPECT_EQ("[00-5e] -> 0\n" + "[5f-5f] -> 1\n" + "[60-ff] -> 0\n", + bytemap); +} + +TEST(TestCompile, UTF8Ranges) { + // The distinct byte ranges involved in the UTF-8 dot ([^\n]). + // Once, erroneously split between 0x3f and 0x40 because it is + // a 6-bit boundary. + + std::string bytemap; + + DumpByteMap(".", Regexp::PerlX, &bytemap); + EXPECT_EQ("[00-09] -> 0\n" + "[0a-0a] -> 1\n" + "[0b-7f] -> 0\n" + "[80-8f] -> 2\n" + "[90-9f] -> 3\n" + "[a0-bf] -> 4\n" + "[c0-c1] -> 1\n" + "[c2-df] -> 5\n" + "[e0-e0] -> 6\n" + "[e1-ef] -> 7\n" + "[f0-f0] -> 8\n" + "[f1-f3] -> 9\n" + "[f4-f4] -> 10\n" + "[f5-ff] -> 1\n", + bytemap); +} + +TEST(TestCompile, InsufficientMemory) { + Regexp* re = Regexp::Parse( + "^(?P[^\\s]+)\\s+(?P[^\\s]+)\\s+(?P.+)$", + Regexp::LikePerl, NULL); + EXPECT_TRUE(re != NULL); + Prog* prog = re->CompileToProg(920); + // If the memory budget has been exhausted, compilation should fail + // and return NULL instead of trying to do anything with NoMatch(). + EXPECT_TRUE(prog == NULL); + re->Decref(); +} + +static void Dump(StringPiece pattern, Regexp::ParseFlags flags, + std::string* forward, std::string* reverse) { + Regexp* re = Regexp::Parse(pattern, flags, NULL); + EXPECT_TRUE(re != NULL); + + if (forward != NULL) { + Prog* prog = re->CompileToProg(0); + EXPECT_TRUE(prog != NULL); + *forward = prog->Dump(); + delete prog; + } + + if (reverse != NULL) { + Prog* prog = re->CompileToReverseProg(0); + EXPECT_TRUE(prog != NULL); + *reverse = prog->Dump(); + delete prog; + } + + re->Decref(); +} + +TEST(TestCompile, Bug26705922) { + // Bug in the compiler caused inefficient bytecode to be generated for Unicode + // groups: common suffixes were cached, but common prefixes were not factored. + + std::string forward, reverse; + + Dump("[\\x{10000}\\x{10010}]", Regexp::LikePerl, &forward, &reverse); + EXPECT_EQ("3. byte [f0-f0] 0 -> 4\n" + "4. byte [90-90] 0 -> 5\n" + "5. byte [80-80] 0 -> 6\n" + "6+ byte [80-80] 0 -> 8\n" + "7. byte [90-90] 0 -> 8\n" + "8. match! 0\n", + forward); + EXPECT_EQ("3+ byte [80-80] 0 -> 5\n" + "4. byte [90-90] 0 -> 5\n" + "5. byte [80-80] 0 -> 6\n" + "6. byte [90-90] 0 -> 7\n" + "7. byte [f0-f0] 0 -> 8\n" + "8. match! 0\n", + reverse); + + Dump("[\\x{8000}-\\x{10FFF}]", Regexp::LikePerl, &forward, &reverse); + EXPECT_EQ("3+ byte [e8-ef] 0 -> 5\n" + "4. byte [f0-f0] 0 -> 8\n" + "5. byte [80-bf] 0 -> 6\n" + "6. byte [80-bf] 0 -> 7\n" + "7. match! 0\n" + "8. byte [90-90] 0 -> 5\n", + forward); + EXPECT_EQ("3. byte [80-bf] 0 -> 4\n" + "4. byte [80-bf] 0 -> 5\n" + "5+ byte [e8-ef] 0 -> 7\n" + "6. byte [90-90] 0 -> 8\n" + "7. match! 0\n" + "8. byte [f0-f0] 0 -> 7\n", + reverse); + + Dump("[\\x{80}-\\x{10FFFF}]", Regexp::LikePerl, NULL, &reverse); + EXPECT_EQ("3. byte [80-bf] 0 -> 4\n" + "4+ byte [c2-df] 0 -> 7\n" + "5+ byte [a0-bf] 1 -> 8\n" + "6. byte [80-bf] 0 -> 9\n" + "7. match! 0\n" + "8. byte [e0-e0] 0 -> 7\n" + "9+ byte [e1-ef] 0 -> 7\n" + "10+ byte [90-bf] 1 -> 13\n" + "11+ byte [80-bf] 1 -> 14\n" + "12. byte [80-8f] 0 -> 15\n" + "13. byte [f0-f0] 0 -> 7\n" + "14. byte [f1-f3] 0 -> 7\n" + "15. byte [f4-f4] 0 -> 7\n", + reverse); +} + +TEST(TestCompile, Bug35237384) { + // Bug in the compiler caused inefficient bytecode to be generated for + // nested nullable subexpressions. + + std::string forward; + + Dump("a**{3,}", Regexp::Latin1|Regexp::NeverCapture, &forward, NULL); + EXPECT_EQ("3+ byte [61-61] 1 -> 3\n" + "4. nop -> 5\n" + "5+ byte [61-61] 1 -> 5\n" + "6. nop -> 7\n" + "7+ byte [61-61] 1 -> 7\n" + "8. match! 0\n", + forward); + + Dump("(a*|b*)*{3,}", Regexp::Latin1|Regexp::NeverCapture, &forward, NULL); + EXPECT_EQ("3+ nop -> 6\n" + "4+ nop -> 8\n" + "5. nop -> 21\n" + "6+ byte [61-61] 1 -> 6\n" + "7. nop -> 3\n" + "8+ byte [62-62] 1 -> 8\n" + "9. nop -> 3\n" + "10+ byte [61-61] 1 -> 10\n" + "11. nop -> 21\n" + "12+ byte [62-62] 1 -> 12\n" + "13. nop -> 21\n" + "14+ byte [61-61] 1 -> 14\n" + "15. nop -> 18\n" + "16+ byte [62-62] 1 -> 16\n" + "17. nop -> 18\n" + "18+ nop -> 14\n" + "19+ nop -> 16\n" + "20. match! 0\n" + "21+ nop -> 10\n" + "22+ nop -> 12\n" + "23. nop -> 18\n", + forward); + + Dump("((|S.+)+|(|S.+)+|){2}", Regexp::Latin1|Regexp::NeverCapture, &forward, NULL); + EXPECT_EQ("3+ nop -> 36\n" + "4+ nop -> 31\n" + "5. nop -> 33\n" + "6+ byte [00-09] 0 -> 8\n" + "7. byte [0b-ff] 0 -> 8\n" + "8+ nop -> 6\n" + "9+ nop -> 29\n" + "10. nop -> 28\n" + "11+ byte [00-09] 0 -> 13\n" + "12. byte [0b-ff] 0 -> 13\n" + "13+ nop -> 11\n" + "14+ nop -> 26\n" + "15. nop -> 28\n" + "16+ byte [00-09] 0 -> 18\n" + "17. byte [0b-ff] 0 -> 18\n" + "18+ nop -> 16\n" + "19+ nop -> 36\n" + "20. nop -> 33\n" + "21+ byte [00-09] 0 -> 23\n" + "22. byte [0b-ff] 0 -> 23\n" + "23+ nop -> 21\n" + "24+ nop -> 31\n" + "25. nop -> 33\n" + "26+ nop -> 28\n" + "27. byte [53-53] 0 -> 11\n" + "28. match! 0\n" + "29+ nop -> 28\n" + "30. byte [53-53] 0 -> 6\n" + "31+ nop -> 33\n" + "32. byte [53-53] 0 -> 21\n" + "33+ nop -> 29\n" + "34+ nop -> 26\n" + "35. nop -> 28\n" + "36+ nop -> 33\n" + "37. byte [53-53] 0 -> 16\n", + forward); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/dfa_test.cc b/extern/re2/re2/testing/dfa_test.cc new file mode 100644 index 0000000000..fb3cc14547 --- /dev/null +++ b/extern/re2/re2/testing/dfa_test.cc @@ -0,0 +1,381 @@ +// Copyright 2006-2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include +#include +#include + +#include "util/test.h" +#include "util/logging.h" +#include "util/strutil.h" +#include "re2/prog.h" +#include "re2/re2.h" +#include "re2/regexp.h" +#include "re2/testing/regexp_generator.h" +#include "re2/testing/string_generator.h" + +static const bool UsingMallocCounter = false; + +DEFINE_int32(size, 8, "log2(number of DFA nodes)"); +DEFINE_int32(repeat, 2, "Repetition count."); +DEFINE_int32(threads, 4, "number of threads"); + +namespace re2 { + +// Check that multithreaded access to DFA class works. + +// Helper function: builds entire DFA for prog. +static void DoBuild(Prog* prog) { + ASSERT_TRUE(prog->BuildEntireDFA(Prog::kFirstMatch, nullptr)); +} + +TEST(Multithreaded, BuildEntireDFA) { + // Create regexp with 2^FLAGS_size states in DFA. + std::string s = "a"; + for (int i = 0; i < FLAGS_size; i++) + s += "[ab]"; + s += "b"; + Regexp* re = Regexp::Parse(s, Regexp::LikePerl, NULL); + ASSERT_TRUE(re != NULL); + + // Check that single-threaded code works. + { + Prog* prog = re->CompileToProg(0); + ASSERT_TRUE(prog != NULL); + + std::thread t(DoBuild, prog); + t.join(); + + delete prog; + } + + // Build the DFA simultaneously in a bunch of threads. + for (int i = 0; i < FLAGS_repeat; i++) { + Prog* prog = re->CompileToProg(0); + ASSERT_TRUE(prog != NULL); + + std::vector threads; + for (int j = 0; j < FLAGS_threads; j++) + threads.emplace_back(DoBuild, prog); + for (int j = 0; j < FLAGS_threads; j++) + threads[j].join(); + + // One more compile, to make sure everything is okay. + prog->BuildEntireDFA(Prog::kFirstMatch, nullptr); + delete prog; + } + + re->Decref(); +} + +// Check that DFA size requirements are followed. +// BuildEntireDFA will, like SearchDFA, stop building out +// the DFA once the memory limits are reached. +TEST(SingleThreaded, BuildEntireDFA) { + // Create regexp with 2^30 states in DFA. + Regexp* re = Regexp::Parse("a[ab]{30}b", Regexp::LikePerl, NULL); + ASSERT_TRUE(re != NULL); + + for (int i = 17; i < 24; i++) { + int64_t limit = int64_t{1}<CompileToProg(limit); + ASSERT_TRUE(prog != NULL); + //progusage = m.HeapGrowth(); + //dfamem = prog->dfa_mem(); + prog->BuildEntireDFA(Prog::kFirstMatch, nullptr); + prog->BuildEntireDFA(Prog::kLongestMatch, nullptr); + usage = m.HeapGrowth(); + delete prog; + } + if (UsingMallocCounter) { + //LOG(INFO) << "limit " << limit << ", " + // << "prog usage " << progusage << ", " + // << "DFA budget " << dfamem << ", " + // << "total " << usage; + // Tolerate +/- 10%. + ASSERT_GT(usage, limit*9/10); + ASSERT_LT(usage, limit*11/10); + } + } + re->Decref(); +} + +// Generates and returns a string over binary alphabet {0,1} that contains +// all possible binary sequences of length n as subsequences. The obvious +// brute force method would generate a string of length n * 2^n, but this +// generates a string of length n + 2^n - 1 called a De Bruijn cycle. +// See Knuth, The Art of Computer Programming, Vol 2, Exercise 3.2.2 #17. +// Such a string is useful for testing a DFA. If you have a DFA +// where distinct last n bytes implies distinct states, then running on a +// DeBruijn string causes the DFA to need to create a new state at every +// position in the input, never reusing any states until it gets to the +// end of the string. This is the worst possible case for DFA execution. +static std::string DeBruijnString(int n) { + CHECK_LT(n, static_cast(8*sizeof(int))); + CHECK_GT(n, 0); + + std::vector did(size_t{1}<CompileToProg(1<SearchDFA(match, StringPiece(), Prog::kUnanchored, + Prog::kFirstMatch, NULL, &failed, NULL); + ASSERT_FALSE(failed); + ASSERT_TRUE(matched); + matched = prog->SearchDFA(no_match, StringPiece(), Prog::kUnanchored, + Prog::kFirstMatch, NULL, &failed, NULL); + ASSERT_FALSE(failed); + ASSERT_FALSE(matched); + } + usage = m.HeapGrowth(); + peak_usage = m.PeakHeapGrowth(); + delete prog; + } + if (UsingMallocCounter) { + //LOG(INFO) << "usage " << usage << ", " + // << "peak usage " << peak_usage; + ASSERT_LT(usage, 1<Decref(); + + // Reset to original behaviour. + Prog::TEST_dfa_should_bail_when_slow(true); +} + +// Helper function: searches for match, which should match, +// and no_match, which should not. +static void DoSearch(Prog* prog, const StringPiece& match, + const StringPiece& no_match) { + for (int i = 0; i < 2; i++) { + bool matched = false; + bool failed = false; + matched = prog->SearchDFA(match, StringPiece(), Prog::kUnanchored, + Prog::kFirstMatch, NULL, &failed, NULL); + ASSERT_FALSE(failed); + ASSERT_TRUE(matched); + matched = prog->SearchDFA(no_match, StringPiece(), Prog::kUnanchored, + Prog::kFirstMatch, NULL, &failed, NULL); + ASSERT_FALSE(failed); + ASSERT_FALSE(matched); + } +} + +TEST(Multithreaded, SearchDFA) { + Prog::TEST_dfa_should_bail_when_slow(false); + + // Same as single-threaded test above. + const int n = 18; + Regexp* re = Regexp::Parse(StringPrintf("0[01]{%d}$", n), + Regexp::LikePerl, NULL); + ASSERT_TRUE(re != NULL); + std::string no_match = DeBruijnString(n); + std::string match = no_match + "0"; + + // Check that single-threaded code works. + { + Prog* prog = re->CompileToProg(1<CompileToProg(1< threads; + for (int j = 0; j < FLAGS_threads; j++) + threads.emplace_back(DoSearch, prog, match, no_match); + for (int j = 0; j < FLAGS_threads; j++) + threads[j].join(); + + delete prog; + } + + re->Decref(); + + // Reset to original behaviour. + Prog::TEST_dfa_should_bail_when_slow(true); +} + +struct ReverseTest { + const char* regexp; + const char* text; + bool match; +}; + +// Test that reverse DFA handles anchored/unanchored correctly. +// It's in the DFA interface but not used by RE2. +ReverseTest reverse_tests[] = { + { "\\A(a|b)", "abc", true }, + { "(a|b)\\z", "cba", true }, + { "\\A(a|b)", "cba", false }, + { "(a|b)\\z", "abc", false }, +}; + +TEST(DFA, ReverseMatch) { + int nfail = 0; + for (size_t i = 0; i < arraysize(reverse_tests); i++) { + const ReverseTest& t = reverse_tests[i]; + Regexp* re = Regexp::Parse(t.regexp, Regexp::LikePerl, NULL); + ASSERT_TRUE(re != NULL); + Prog* prog = re->CompileToReverseProg(0); + ASSERT_TRUE(prog != NULL); + bool failed = false; + bool matched = prog->SearchDFA(t.text, StringPiece(), Prog::kUnanchored, + Prog::kFirstMatch, NULL, &failed, NULL); + if (matched != t.match) { + LOG(ERROR) << t.regexp << " on " << t.text << ": want " << t.match; + nfail++; + } + delete prog; + re->Decref(); + } + EXPECT_EQ(nfail, 0); +} + +struct CallbackTest { + const char* regexp; + const char* dump; +}; + +// Test that DFA::BuildAllStates() builds the expected DFA states +// and issues the expected callbacks. These test cases reflect the +// very compact encoding of the callbacks, but that also makes them +// very difficult to understand, so let's work through "\\Aa\\z". +// There are three slots per DFA state because the bytemap has two +// equivalence classes and there is a third slot for kByteEndText: +// 0: all bytes that are not 'a' +// 1: the byte 'a' +// 2: kByteEndText +// -1 means that there is no transition from that DFA state to any +// other DFA state for that slot. The valid transitions are thus: +// state 0 --slot 1--> state 1 +// state 1 --slot 2--> state 2 +// The double brackets indicate that state 2 is a matching state. +// Putting it together, this means that the DFA must consume the +// byte 'a' and then hit end of text. Q.E.D. +CallbackTest callback_tests[] = { + { "\\Aa\\z", "[-1,1,-1] [-1,-1,2] [[-1,-1,-1]]" }, + { "\\Aab\\z", "[-1,1,-1,-1] [-1,-1,2,-1] [-1,-1,-1,3] [[-1,-1,-1,-1]]" }, + { "\\Aa*b\\z", "[-1,0,1,-1] [-1,-1,-1,2] [[-1,-1,-1,-1]]" }, + { "\\Aa+b\\z", "[-1,1,-1,-1] [-1,1,2,-1] [-1,-1,-1,3] [[-1,-1,-1,-1]]" }, + { "\\Aa?b\\z", "[-1,1,2,-1] [-1,-1,2,-1] [-1,-1,-1,3] [[-1,-1,-1,-1]]" }, + { "\\Aa\\C*\\z", "[-1,1,-1] [1,1,2] [[-1,-1,-1]]" }, + { "\\Aa\\C*", "[-1,1,-1] [2,2,3] [[2,2,2]] [[-1,-1,-1]]" }, + { "a\\C*", "[0,1,-1] [2,2,3] [[2,2,2]] [[-1,-1,-1]]" }, + { "\\C*", "[1,2] [[1,1]] [[-1,-1]]" }, + { "a", "[0,1,-1] [2,2,2] [[-1,-1,-1]]"} , +}; + +TEST(DFA, Callback) { + int nfail = 0; + for (size_t i = 0; i < arraysize(callback_tests); i++) { + const CallbackTest& t = callback_tests[i]; + Regexp* re = Regexp::Parse(t.regexp, Regexp::LikePerl, NULL); + ASSERT_TRUE(re != NULL); + Prog* prog = re->CompileToProg(0); + ASSERT_TRUE(prog != NULL); + std::string dump; + prog->BuildEntireDFA(Prog::kLongestMatch, [&](const int* next, bool match) { + ASSERT_TRUE(next != NULL); + if (!dump.empty()) + dump += " "; + dump += match ? "[[" : "["; + for (int b = 0; b < prog->bytemap_range() + 1; b++) + dump += StringPrintf("%d,", next[b]); + dump.pop_back(); + dump += match ? "]]" : "]"; + }); + if (dump != t.dump) { + LOG(ERROR) << t.regexp << " bytemap:\n" << prog->DumpByteMap(); + LOG(ERROR) << t.regexp << " dump:\ngot " << dump << "\nwant " << t.dump; + nfail++; + } + delete prog; + re->Decref(); + } + EXPECT_EQ(nfail, 0); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/dump.cc b/extern/re2/re2/testing/dump.cc new file mode 100644 index 0000000000..1df8ddde6e --- /dev/null +++ b/extern/re2/re2/testing/dump.cc @@ -0,0 +1,169 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Dump the regexp into a string showing structure. +// Tested by parse_unittest.cc + +// This function traverses the regexp recursively, +// meaning that on inputs like Regexp::Simplify of +// a{100}{100}{100}{100}{100}{100}{100}{100}{100}{100}, +// it takes time and space exponential in the size of the +// original regular expression. It can also use stack space +// linear in the size of the regular expression for inputs +// like ((((((((((((((((a*)*)*)*)*)*)*)*)*)*)*)*)*)*)*)*)*. +// IT IS NOT SAFE TO CALL FROM PRODUCTION CODE. +// As a result, Dump is provided only in the testing +// library (see BUILD). + +#include + +#include "util/test.h" +#include "util/logging.h" +#include "util/strutil.h" +#include "util/utf.h" +#include "re2/stringpiece.h" +#include "re2/regexp.h" + +// Cause a link error if this file is used outside of testing. +DECLARE_string(test_tmpdir); + +namespace re2 { + +static const char* kOpcodeNames[] = { + "bad", + "no", + "emp", + "lit", + "str", + "cat", + "alt", + "star", + "plus", + "que", + "rep", + "cap", + "dot", + "byte", + "bol", + "eol", + "wb", // kRegexpWordBoundary + "nwb", // kRegexpNoWordBoundary + "bot", + "eot", + "cc", + "match", +}; + +// Create string representation of regexp with explicit structure. +// Nothing pretty, just for testing. +static void DumpRegexpAppending(Regexp* re, std::string* s) { + if (re->op() < 0 || re->op() >= arraysize(kOpcodeNames)) { + *s += StringPrintf("op%d", re->op()); + } else { + switch (re->op()) { + default: + break; + case kRegexpStar: + case kRegexpPlus: + case kRegexpQuest: + case kRegexpRepeat: + if (re->parse_flags() & Regexp::NonGreedy) + s->append("n"); + break; + } + s->append(kOpcodeNames[re->op()]); + if (re->op() == kRegexpLiteral && (re->parse_flags() & Regexp::FoldCase)) { + Rune r = re->rune(); + if ('a' <= r && r <= 'z') + s->append("fold"); + } + if (re->op() == kRegexpLiteralString && (re->parse_flags() & Regexp::FoldCase)) { + for (int i = 0; i < re->nrunes(); i++) { + Rune r = re->runes()[i]; + if ('a' <= r && r <= 'z') { + s->append("fold"); + break; + } + } + } + } + s->append("{"); + switch (re->op()) { + default: + break; + case kRegexpEndText: + if (!(re->parse_flags() & Regexp::WasDollar)) { + s->append("\\z"); + } + break; + case kRegexpLiteral: { + Rune r = re->rune(); + char buf[UTFmax+1]; + buf[runetochar(buf, &r)] = 0; + s->append(buf); + break; + } + case kRegexpLiteralString: + for (int i = 0; i < re->nrunes(); i++) { + Rune r = re->runes()[i]; + char buf[UTFmax+1]; + buf[runetochar(buf, &r)] = 0; + s->append(buf); + } + break; + case kRegexpConcat: + case kRegexpAlternate: + for (int i = 0; i < re->nsub(); i++) + DumpRegexpAppending(re->sub()[i], s); + break; + case kRegexpStar: + case kRegexpPlus: + case kRegexpQuest: + DumpRegexpAppending(re->sub()[0], s); + break; + case kRegexpCapture: + if (re->cap() == 0) + LOG(DFATAL) << "kRegexpCapture cap() == 0"; + if (re->name()) { + s->append(*re->name()); + s->append(":"); + } + DumpRegexpAppending(re->sub()[0], s); + break; + case kRegexpRepeat: + s->append(StringPrintf("%d,%d ", re->min(), re->max())); + DumpRegexpAppending(re->sub()[0], s); + break; + case kRegexpCharClass: { + std::string sep; + for (CharClass::iterator it = re->cc()->begin(); + it != re->cc()->end(); ++it) { + RuneRange rr = *it; + s->append(sep); + if (rr.lo == rr.hi) + s->append(StringPrintf("%#x", rr.lo)); + else + s->append(StringPrintf("%#x-%#x", rr.lo, rr.hi)); + sep = " "; + } + break; + } + } + s->append("}"); +} + +std::string Regexp::Dump() { + std::string s; + + // Make sure being called from a unit test. + if (FLAGS_test_tmpdir.empty()) { + LOG(ERROR) << "Cannot use except for testing."; + return s; + } + + DumpRegexpAppending(this, &s); + return s; +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/exhaustive1_test.cc b/extern/re2/re2/testing/exhaustive1_test.cc new file mode 100644 index 0000000000..9ead27e646 --- /dev/null +++ b/extern/re2/re2/testing/exhaustive1_test.cc @@ -0,0 +1,44 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Exhaustive testing of regular expression matching. + +#include +#include + +#include "util/test.h" +#include "re2/testing/exhaustive_tester.h" + +DECLARE_string(regexp_engines); + +namespace re2 { + +// Test simple repetition operators +TEST(Repetition, Simple) { + std::vector ops = Split(" ", + "%s{0} %s{0,} %s{1} %s{1,} %s{0,1} %s{0,2} " + "%s{1,2} %s{2} %s{2,} %s{3,4} %s{4,5} " + "%s* %s+ %s? %s*? %s+? %s??"); + ExhaustiveTest(3, 2, Explode("abc."), ops, + 6, Explode("ab"), "(?:%s)", ""); + ExhaustiveTest(3, 2, Explode("abc."), ops, + 40, Explode("a"), "(?:%s)", ""); +} + +// Test capturing parens -- (a) -- inside repetition operators +TEST(Repetition, Capturing) { + std::vector ops = Split(" ", + "%s{0} %s{0,} %s{1} %s{1,} %s{0,1} %s{0,2} " + "%s{1,2} %s{2} %s{2,} %s{3,4} %s{4,5} " + "%s* %s+ %s? %s*? %s+? %s??"); + ExhaustiveTest(3, 2, Split(" ", "a (a) b"), ops, + 7, Explode("ab"), "(?:%s)", ""); + + // This would be a great test, but it runs forever when PCRE is enabled. + if (FLAGS_regexp_engines.find("PCRE") == std::string::npos) + ExhaustiveTest(3, 2, Split(" ", "a (a)"), ops, + 50, Explode("a"), "(?:%s)", ""); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/exhaustive2_test.cc b/extern/re2/re2/testing/exhaustive2_test.cc new file mode 100644 index 0000000000..ce4235b0e5 --- /dev/null +++ b/extern/re2/re2/testing/exhaustive2_test.cc @@ -0,0 +1,73 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Exhaustive testing of regular expression matching. + +#include +#include +#include +#include + +#include "util/test.h" +#include "re2/re2.h" +#include "re2/testing/exhaustive_tester.h" + +namespace re2 { + +// Test empty string matches (aka "(?:)") +TEST(EmptyString, Exhaustive) { + ExhaustiveTest(2, 2, Split(" ", "(?:) a"), + RegexpGenerator::EgrepOps(), + 5, Split("", "ab"), "", ""); +} + +// Test escaped versions of regexp syntax. +TEST(Punctuation, Literals) { + std::vector alphabet = Explode("()*+?{}[]\\^$."); + std::vector escaped = alphabet; + for (size_t i = 0; i < escaped.size(); i++) + escaped[i] = "\\" + escaped[i]; + ExhaustiveTest(1, 1, escaped, RegexpGenerator::EgrepOps(), + 2, alphabet, "", ""); +} + +// Test ^ $ . \A \z in presence of line endings. +// Have to wrap the empty-width ones in (?:) so that +// they can be repeated -- PCRE rejects ^* but allows (?:^)* +TEST(LineEnds, Exhaustive) { + ExhaustiveTest(2, 2, Split(" ", "(?:^) (?:$) . a \\n (?:\\A) (?:\\z)"), + RegexpGenerator::EgrepOps(), + 4, Explode("ab\n"), "", ""); +} + +// Test what does and does not match \n. +// This would be a good test, except that PCRE seems to have a bug: +// in single-byte character set mode (the default), +// [^a] matches \n, but in UTF-8 mode it does not. +// So when we run the test, the tester complains that +// we don't agree with PCRE, but it's PCRE that is at fault. +// For what it's worth, Perl gets this right (matches +// regardless of whether UTF-8 input is selected): +// +// #!/usr/bin/perl +// use POSIX qw(locale_h); +// print "matches in latin1\n" if "\n" =~ /[^a]/; +// setlocale("en_US.utf8"); +// print "matches in utf8\n" if "\n" =~ /[^a]/; +// +// The rule chosen for RE2 is that by default, like Perl, +// dot does not match \n but negated character classes [^a] do. +// (?s) will allow dot to match \n; there is no way in RE2 +// to stop [^a] from matching \n, though the underlying library +// provides a mechanism, and RE2 could add new syntax if needed. +// +// TEST(Newlines, Exhaustive) { +// std::vector empty_vector; +// ExhaustiveTest(1, 1, Split(" ", "\\n . a [^a]"), +// RegexpGenerator::EgrepOps(), +// 4, Explode("a\n"), ""); +// } + +} // namespace re2 + diff --git a/extern/re2/re2/testing/exhaustive3_test.cc b/extern/re2/re2/testing/exhaustive3_test.cc new file mode 100644 index 0000000000..1fe46b6db1 --- /dev/null +++ b/extern/re2/re2/testing/exhaustive3_test.cc @@ -0,0 +1,100 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Exhaustive testing of regular expression matching. + +#include +#include +#include +#include + +#include "util/test.h" +#include "util/utf.h" +#include "re2/testing/exhaustive_tester.h" + +namespace re2 { + +// Test simple character classes by themselves. +TEST(CharacterClasses, Exhaustive) { + std::vector atoms = Split(" ", + "[a] [b] [ab] [^bc] [b-d] [^b-d] []a] [-a] [a-] [^-a] [a-b-c] a b ."); + ExhaustiveTest(2, 1, atoms, RegexpGenerator::EgrepOps(), + 5, Explode("ab"), "", ""); +} + +// Test simple character classes inside a___b (for example, a[a]b). +TEST(CharacterClasses, ExhaustiveAB) { + std::vector atoms = Split(" ", + "[a] [b] [ab] [^bc] [b-d] [^b-d] []a] [-a] [a-] [^-a] [a-b-c] a b ."); + ExhaustiveTest(2, 1, atoms, RegexpGenerator::EgrepOps(), + 5, Explode("ab"), "a%sb", ""); +} + +// Returns UTF8 for Rune r +static std::string UTF8(Rune r) { + char buf[UTFmax+1]; + buf[runetochar(buf, &r)] = 0; + return std::string(buf); +} + +// Returns a vector of "interesting" UTF8 characters. +// Unicode is now too big to just return all of them, +// so UTF8Characters return a set likely to be good test cases. +static const std::vector& InterestingUTF8() { + static bool init; + static std::vector v; + + if (init) + return v; + + init = true; + // All the Latin1 equivalents are interesting. + for (int i = 1; i < 256; i++) + v.push_back(UTF8(i)); + + // After that, the codes near bit boundaries are + // interesting, because they span byte sequence lengths. + for (int j = 0; j < 8; j++) + v.push_back(UTF8(256 + j)); + for (int i = 512; i < Runemax; i <<= 1) + for (int j = -8; j < 8; j++) + v.push_back(UTF8(i + j)); + + // The codes near Runemax, including Runemax itself, are interesting. + for (int j = -8; j <= 0; j++) + v.push_back(UTF8(Runemax + j)); + + return v; +} + +// Test interesting UTF-8 characters against character classes. +TEST(InterestingUTF8, SingleOps) { + std::vector atoms = Split(" ", + ". ^ $ \\a \\f \\n \\r \\t \\v \\d \\D \\s \\S \\w \\W \\b \\B " + "[[:alnum:]] [[:alpha:]] [[:blank:]] [[:cntrl:]] [[:digit:]] " + "[[:graph:]] [[:lower:]] [[:print:]] [[:punct:]] [[:space:]] " + "[[:upper:]] [[:xdigit:]] [\\s\\S] [\\d\\D] [^\\w\\W] [^\\d\\D]"); + std::vector ops; // no ops + ExhaustiveTest(1, 0, atoms, ops, + 1, InterestingUTF8(), "", ""); +} + +// Test interesting UTF-8 characters against character classes, +// but wrap everything inside AB. +TEST(InterestingUTF8, AB) { + std::vector atoms = Split(" ", + ". ^ $ \\a \\f \\n \\r \\t \\v \\d \\D \\s \\S \\w \\W \\b \\B " + "[[:alnum:]] [[:alpha:]] [[:blank:]] [[:cntrl:]] [[:digit:]] " + "[[:graph:]] [[:lower:]] [[:print:]] [[:punct:]] [[:space:]] " + "[[:upper:]] [[:xdigit:]] [\\s\\S] [\\d\\D] [^\\w\\W] [^\\d\\D]"); + std::vector ops; // no ops + std::vector alpha = InterestingUTF8(); + for (size_t i = 0; i < alpha.size(); i++) + alpha[i] = "a" + alpha[i] + "b"; + ExhaustiveTest(1, 0, atoms, ops, + 1, alpha, "a%sb", ""); +} + +} // namespace re2 + diff --git a/extern/re2/re2/testing/exhaustive_test.cc b/extern/re2/re2/testing/exhaustive_test.cc new file mode 100644 index 0000000000..514fd9092f --- /dev/null +++ b/extern/re2/re2/testing/exhaustive_test.cc @@ -0,0 +1,36 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Exhaustive testing of regular expression matching. + +#include "util/test.h" +#include "re2/testing/exhaustive_tester.h" + +namespace re2 { + +// Test very simple expressions. +TEST(EgrepLiterals, Lowercase) { + EgrepTest(3, 2, "abc.", 3, "abc", ""); +} + +// Test mixed-case expressions. +TEST(EgrepLiterals, MixedCase) { + EgrepTest(3, 2, "AaBb.", 2, "AaBb", ""); +} + +// Test mixed-case in case-insensitive mode. +TEST(EgrepLiterals, FoldCase) { + // The punctuation characters surround A-Z and a-z + // in the ASCII table. This looks for bugs in the + // bytemap range code in the DFA. + EgrepTest(3, 2, "abAB.", 2, "aBc@_~", "(?i:%s)"); +} + +// Test very simple expressions. +TEST(EgrepLiterals, UTF8) { + EgrepTest(3, 2, "ab.", 4, "a\xE2\x98\xBA", ""); +} + +} // namespace re2 + diff --git a/extern/re2/re2/testing/exhaustive_tester.cc b/extern/re2/re2/testing/exhaustive_tester.cc new file mode 100644 index 0000000000..47950ba711 --- /dev/null +++ b/extern/re2/re2/testing/exhaustive_tester.cc @@ -0,0 +1,188 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Exhaustive testing of regular expression matching. + +// Each test picks an alphabet (e.g., "abc"), a maximum string length, +// a maximum regular expression length, and a maximum number of letters +// that can appear in the regular expression. Given these parameters, +// it tries every possible regular expression and string, verifying that +// the NFA, DFA, and a trivial backtracking implementation agree about +// the location of the match. + +#include + +#include "util/test.h" +#include "util/logging.h" +#include "util/strutil.h" +#include "re2/testing/exhaustive_tester.h" +#include "re2/testing/tester.h" + +// For target `log' in the Makefile. +#ifndef LOGGING +#define LOGGING 0 +#endif + +DEFINE_bool(show_regexps, false, "show regexps during testing"); + +DEFINE_int32(max_bad_regexp_inputs, 1, + "Stop testing a regular expression after finding this many " + "strings that break it."); + +namespace re2 { + +static char* escape(const StringPiece& sp) { + static char buf[512]; + char* p = buf; + *p++ = '\"'; + for (size_t i = 0; i < sp.size(); i++) { + if(p+5 >= buf+sizeof buf) + LOG(FATAL) << "ExhaustiveTester escape: too long"; + if(sp[i] == '\\' || sp[i] == '\"') { + *p++ = '\\'; + *p++ = sp[i]; + } else if(sp[i] == '\n') { + *p++ = '\\'; + *p++ = 'n'; + } else { + *p++ = sp[i]; + } + } + *p++ = '\"'; + *p = '\0'; + return buf; +} + +static void PrintResult(const RE2& re, const StringPiece& input, RE2::Anchor anchor, StringPiece *m, int n) { + if (!re.Match(input, 0, input.size(), anchor, m, n)) { + printf("-"); + return; + } + for (int i = 0; i < n; i++) { + if (i > 0) + printf(" "); + if (m[i].begin() == NULL) + printf("-"); + else + printf("%td-%td", + m[i].begin() - input.begin(), m[i].end() - input.begin()); + } +} + +// Processes a single generated regexp. +// Compiles it using Regexp interface and PCRE, and then +// checks that NFA, DFA, and PCRE all return the same results. +void ExhaustiveTester::HandleRegexp(const std::string& const_regexp) { + regexps_++; + std::string regexp = const_regexp; + if (!topwrapper_.empty()) + regexp = StringPrintf(topwrapper_.c_str(), regexp.c_str()); + + if (FLAGS_show_regexps) { + printf("\r%s", regexp.c_str()); + fflush(stdout); + } + + if (LOGGING) { + // Write out test cases and answers for use in testing + // other implementations, such as Go's regexp package. + if (randomstrings_) + LOG(ERROR) << "Cannot log with random strings."; + if (regexps_ == 1) { // first + printf("strings\n"); + strgen_.Reset(); + while (strgen_.HasNext()) + printf("%s\n", escape(strgen_.Next())); + printf("regexps\n"); + } + printf("%s\n", escape(regexp)); + + RE2 re(regexp); + RE2::Options longest; + longest.set_longest_match(true); + RE2 relongest(regexp, longest); + int ngroup = re.NumberOfCapturingGroups()+1; + StringPiece* group = new StringPiece[ngroup]; + + strgen_.Reset(); + while (strgen_.HasNext()) { + StringPiece input = strgen_.Next(); + PrintResult(re, input, RE2::ANCHOR_BOTH, group, ngroup); + printf(";"); + PrintResult(re, input, RE2::UNANCHORED, group, ngroup); + printf(";"); + PrintResult(relongest, input, RE2::ANCHOR_BOTH, group, ngroup); + printf(";"); + PrintResult(relongest, input, RE2::UNANCHORED, group, ngroup); + printf("\n"); + } + delete[] group; + return; + } + + Tester tester(regexp); + if (tester.error()) + return; + + strgen_.Reset(); + strgen_.GenerateNULL(); + if (randomstrings_) + strgen_.Random(stringseed_, stringcount_); + int bad_inputs = 0; + while (strgen_.HasNext()) { + tests_++; + if (!tester.TestInput(strgen_.Next())) { + failures_++; + if (++bad_inputs >= FLAGS_max_bad_regexp_inputs) + break; + } + } +} + +// Runs an exhaustive test on the given parameters. +void ExhaustiveTest(int maxatoms, int maxops, + const std::vector& alphabet, + const std::vector& ops, + int maxstrlen, + const std::vector& stralphabet, + const std::string& wrapper, + const std::string& topwrapper) { + if (RE2_DEBUG_MODE) { + if (maxatoms > 1) + maxatoms--; + if (maxops > 1) + maxops--; + if (maxstrlen > 1) + maxstrlen--; + } + ExhaustiveTester t(maxatoms, maxops, alphabet, ops, + maxstrlen, stralphabet, wrapper, + topwrapper); + t.Generate(); + if (!LOGGING) { + printf("%d regexps, %d tests, %d failures [%d/%d str]\n", + t.regexps(), t.tests(), t.failures(), maxstrlen, (int)stralphabet.size()); + } + EXPECT_EQ(0, t.failures()); +} + +// Runs an exhaustive test using the given parameters and +// the basic egrep operators. +void EgrepTest(int maxatoms, int maxops, const std::string& alphabet, + int maxstrlen, const std::string& stralphabet, + const std::string& wrapper) { + const char* tops[] = { "", "^(?:%s)", "(?:%s)$", "^(?:%s)$" }; + + for (size_t i = 0; i < arraysize(tops); i++) { + ExhaustiveTest(maxatoms, maxops, + Split("", alphabet), + RegexpGenerator::EgrepOps(), + maxstrlen, + Split("", stralphabet), + wrapper, + tops[i]); + } +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/exhaustive_tester.h b/extern/re2/re2/testing/exhaustive_tester.h new file mode 100644 index 0000000000..3a14282f01 --- /dev/null +++ b/extern/re2/re2/testing/exhaustive_tester.h @@ -0,0 +1,105 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_TESTING_EXHAUSTIVE_TESTER_H_ +#define RE2_TESTING_EXHAUSTIVE_TESTER_H_ + +#include +#include +#include + +#include "util/util.h" +#include "re2/testing/regexp_generator.h" +#include "re2/testing/string_generator.h" + +namespace re2 { + +// Doing this simplifies the logic below. +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#if !defined(NDEBUG) +// We are in a debug build. +const bool RE2_DEBUG_MODE = true; +#elif __has_feature(address_sanitizer) || __has_feature(memory_sanitizer) || __has_feature(thread_sanitizer) +// Not a debug build, but still under sanitizers. +const bool RE2_DEBUG_MODE = true; +#else +const bool RE2_DEBUG_MODE = false; +#endif + +// Exhaustive regular expression test: generate all regexps within parameters, +// then generate all strings of a given length over a given alphabet, +// then check that NFA, DFA, and PCRE agree about whether each regexp matches +// each possible string, and if so, where the match is. +// +// Can also be used in a "random" mode that generates a given number +// of random regexp and strings, allowing testing of larger expressions +// and inputs. +class ExhaustiveTester : public RegexpGenerator { + public: + ExhaustiveTester(int maxatoms, + int maxops, + const std::vector& alphabet, + const std::vector& ops, + int maxstrlen, + const std::vector& stralphabet, + const std::string& wrapper, + const std::string& topwrapper) + : RegexpGenerator(maxatoms, maxops, alphabet, ops), + strgen_(maxstrlen, stralphabet), + wrapper_(wrapper), + topwrapper_(topwrapper), + regexps_(0), tests_(0), failures_(0), + randomstrings_(0), stringseed_(0), stringcount_(0) { } + + int regexps() { return regexps_; } + int tests() { return tests_; } + int failures() { return failures_; } + + // Needed for RegexpGenerator interface. + void HandleRegexp(const std::string& regexp); + + // Causes testing to generate random input strings. + void RandomStrings(int32_t seed, int32_t count) { + randomstrings_ = true; + stringseed_ = seed; + stringcount_ = count; + } + + private: + StringGenerator strgen_; + std::string wrapper_; // Regexp wrapper - either empty or has one %s. + std::string topwrapper_; // Regexp top-level wrapper. + int regexps_; // Number of HandleRegexp calls + int tests_; // Number of regexp tests. + int failures_; // Number of tests failed. + + bool randomstrings_; // Whether to use random strings + int32_t stringseed_; // If so, the seed. + int stringcount_; // If so, how many to generate. + + ExhaustiveTester(const ExhaustiveTester&) = delete; + ExhaustiveTester& operator=(const ExhaustiveTester&) = delete; +}; + +// Runs an exhaustive test on the given parameters. +void ExhaustiveTest(int maxatoms, int maxops, + const std::vector& alphabet, + const std::vector& ops, + int maxstrlen, + const std::vector& stralphabet, + const std::string& wrapper, + const std::string& topwrapper); + +// Runs an exhaustive test using the given parameters and +// the basic egrep operators. +void EgrepTest(int maxatoms, int maxops, const std::string& alphabet, + int maxstrlen, const std::string& stralphabet, + const std::string& wrapper); + +} // namespace re2 + +#endif // RE2_TESTING_EXHAUSTIVE_TESTER_H_ diff --git a/extern/re2/re2/testing/filtered_re2_test.cc b/extern/re2/re2/testing/filtered_re2_test.cc new file mode 100644 index 0000000000..deef2f87d6 --- /dev/null +++ b/extern/re2/re2/testing/filtered_re2_test.cc @@ -0,0 +1,294 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include +#include +#include +#include + +#include "util/test.h" +#include "util/logging.h" +#include "re2/filtered_re2.h" +#include "re2/re2.h" + +namespace re2 { + +struct FilterTestVars { + FilterTestVars() {} + explicit FilterTestVars(int min_atom_len) : f(min_atom_len) {} + + std::vector atoms; + std::vector atom_indices; + std::vector matches; + RE2::Options opts; + FilteredRE2 f; +}; + +TEST(FilteredRE2Test, EmptyTest) { + FilterTestVars v; + + v.f.Compile(&v.atoms); + EXPECT_EQ(0, v.atoms.size()); + + // Compile has no effect at all when called before Add: it will not + // record that it has been called and it will not clear the vector. + // The second point does not matter here, but the first point means + // that an error will be logged during the call to AllMatches. + v.f.AllMatches("foo", v.atom_indices, &v.matches); + EXPECT_EQ(0, v.matches.size()); +} + +TEST(FilteredRE2Test, SmallOrTest) { + FilterTestVars v(4); // override the minimum atom length + int id; + v.f.Add("(foo|bar)", v.opts, &id); + + v.f.Compile(&v.atoms); + EXPECT_EQ(0, v.atoms.size()); + + v.f.AllMatches("lemurs bar", v.atom_indices, &v.matches); + EXPECT_EQ(1, v.matches.size()); + EXPECT_EQ(id, v.matches[0]); +} + +TEST(FilteredRE2Test, SmallLatinTest) { + FilterTestVars v; + int id; + + v.opts.set_encoding(RE2::Options::EncodingLatin1); + v.f.Add("\xde\xadQ\xbe\xef", v.opts, &id); + v.f.Compile(&v.atoms); + EXPECT_EQ(1, v.atoms.size()); + EXPECT_EQ(v.atoms[0], "\xde\xadq\xbe\xef"); + + v.atom_indices.push_back(0); + v.f.AllMatches("foo\xde\xadQ\xbe\xeflemur", v.atom_indices, &v.matches); + EXPECT_EQ(1, v.matches.size()); + EXPECT_EQ(id, v.matches[0]); +} + +struct AtomTest { + const char* testname; + // If any test needs more than this many regexps or atoms, increase + // the size of the corresponding array. + const char* regexps[20]; + const char* atoms[20]; +}; + +AtomTest atom_tests[] = { + { + // This test checks to make sure empty patterns are allowed. + "CheckEmptyPattern", + {""}, + {} + }, { + // This test checks that all atoms of length greater than min length + // are found, and no atoms that are of smaller length are found. + "AllAtomsGtMinLengthFound", { + "(abc123|def456|ghi789).*mnop[x-z]+", + "abc..yyy..zz", + "mnmnpp[a-z]+PPP" + }, { + "abc123", + "def456", + "ghi789", + "mnop", + "abc", + "yyy", + "mnmnpp", + "ppp" + } + }, { + // Test to make sure that any atoms that have another atom as a + // substring in an OR are removed; that is, only the shortest + // substring is kept. + "SubstrAtomRemovesSuperStrInOr", { + "(abc123|abc|ghi789|abc1234).*[x-z]+", + "abcd..yyy..yyyzzz", + "mnmnpp[a-z]+PPP" + }, { + "abc", + "ghi789", + "abcd", + "yyy", + "yyyzzz", + "mnmnpp", + "ppp" + } + }, { + // Test character class expansion. + "CharClassExpansion", { + "m[a-c][d-f]n.*[x-z]+", + "[x-y]bcde[ab]" + }, { + "madn", "maen", "mafn", + "mbdn", "mben", "mbfn", + "mcdn", "mcen", "mcfn", + "xbcdea", "xbcdeb", + "ybcdea", "ybcdeb" + } + }, { + // Test upper/lower of non-ASCII. + "UnicodeLower", { + "(?i)ΔδΠϖπΣςσ", + "ΛΜΝΟΠ", + "ψρστυ", + }, { + "δδπππσσσ", + "λμνοπ", + "ψρστυ", + }, + }, +}; + +void AddRegexpsAndCompile(const char* regexps[], + size_t n, + struct FilterTestVars* v) { + for (size_t i = 0; i < n; i++) { + int id; + v->f.Add(regexps[i], v->opts, &id); + } + v->f.Compile(&v->atoms); +} + +bool CheckExpectedAtoms(const char* atoms[], + size_t n, + const char* testname, + struct FilterTestVars* v) { + std::vector expected; + for (size_t i = 0; i < n; i++) + expected.push_back(atoms[i]); + + bool pass = expected.size() == v->atoms.size(); + + std::sort(v->atoms.begin(), v->atoms.end()); + std::sort(expected.begin(), expected.end()); + for (size_t i = 0; pass && i < n; i++) + pass = pass && expected[i] == v->atoms[i]; + + if (!pass) { + LOG(ERROR) << "Failed " << testname; + LOG(ERROR) << "Expected #atoms = " << expected.size(); + for (size_t i = 0; i < expected.size(); i++) + LOG(ERROR) << expected[i]; + LOG(ERROR) << "Found #atoms = " << v->atoms.size(); + for (size_t i = 0; i < v->atoms.size(); i++) + LOG(ERROR) << v->atoms[i]; + } + + return pass; +} + +TEST(FilteredRE2Test, AtomTests) { + int nfail = 0; + for (size_t i = 0; i < arraysize(atom_tests); i++) { + FilterTestVars v; + AtomTest* t = &atom_tests[i]; + size_t nregexp, natom; + for (nregexp = 0; nregexp < arraysize(t->regexps); nregexp++) + if (t->regexps[nregexp] == NULL) + break; + for (natom = 0; natom < arraysize(t->atoms); natom++) + if (t->atoms[natom] == NULL) + break; + AddRegexpsAndCompile(t->regexps, nregexp, &v); + if (!CheckExpectedAtoms(t->atoms, natom, t->testname, &v)) + nfail++; + } + EXPECT_EQ(0, nfail); +} + +void FindAtomIndices(const std::vector& atoms, + const std::vector& matched_atoms, + std::vector* atom_indices) { + atom_indices->clear(); + for (size_t i = 0; i < matched_atoms.size(); i++) { + for (size_t j = 0; j < atoms.size(); j++) { + if (matched_atoms[i] == atoms[j]) { + atom_indices->push_back(static_cast(j)); + break; + } + } + } +} + +TEST(FilteredRE2Test, MatchEmptyPattern) { + FilterTestVars v; + AtomTest* t = &atom_tests[0]; + // We are using the regexps used in one of the atom tests + // for this test. Adding the EXPECT here to make sure + // the index we use for the test is for the correct test. + EXPECT_EQ("CheckEmptyPattern", std::string(t->testname)); + size_t nregexp; + for (nregexp = 0; nregexp < arraysize(t->regexps); nregexp++) + if (t->regexps[nregexp] == NULL) + break; + AddRegexpsAndCompile(t->regexps, nregexp, &v); + std::string text = "0123"; + std::vector atom_ids; + std::vector matching_regexps; + EXPECT_EQ(0, v.f.FirstMatch(text, atom_ids)); +} + +TEST(FilteredRE2Test, MatchTests) { + FilterTestVars v; + AtomTest* t = &atom_tests[2]; + // We are using the regexps used in one of the atom tests + // for this test. + EXPECT_EQ("SubstrAtomRemovesSuperStrInOr", std::string(t->testname)); + size_t nregexp; + for (nregexp = 0; nregexp < arraysize(t->regexps); nregexp++) + if (t->regexps[nregexp] == NULL) + break; + AddRegexpsAndCompile(t->regexps, nregexp, &v); + + std::string text = "abc121212xyz"; + // atoms = abc + std::vector atom_ids; + std::vector atoms; + atoms.push_back("abc"); + FindAtomIndices(v.atoms, atoms, &atom_ids); + std::vector matching_regexps; + v.f.AllMatches(text, atom_ids, &matching_regexps); + EXPECT_EQ(1, matching_regexps.size()); + + text = "abc12312yyyzzz"; + atoms.clear(); + atoms.push_back("abc"); + atoms.push_back("yyy"); + atoms.push_back("yyyzzz"); + FindAtomIndices(v.atoms, atoms, &atom_ids); + v.f.AllMatches(text, atom_ids, &matching_regexps); + EXPECT_EQ(1, matching_regexps.size()); + + text = "abcd12yyy32yyyzzz"; + atoms.clear(); + atoms.push_back("abc"); + atoms.push_back("abcd"); + atoms.push_back("yyy"); + atoms.push_back("yyyzzz"); + FindAtomIndices(v.atoms, atoms, &atom_ids); + LOG(INFO) << "S: " << atom_ids.size(); + for (size_t i = 0; i < atom_ids.size(); i++) + LOG(INFO) << "i: " << i << " : " << atom_ids[i]; + v.f.AllMatches(text, atom_ids, &matching_regexps); + EXPECT_EQ(2, matching_regexps.size()); +} + +TEST(FilteredRE2Test, EmptyStringInStringSetBug) { + // Bug due to find() finding "" at the start of everything in a string + // set and thus SimplifyStringSet() would end up erasing everything. + // In order to test this, we have to keep PrefilterTree from discarding + // the OR entirely, so we have to make the minimum atom length zero. + + FilterTestVars v(0); // override the minimum atom length + const char* regexps[] = {"-R.+(|ADD=;AA){12}}"}; + const char* atoms[] = {"", "-r", "add=;aa", "}"}; + AddRegexpsAndCompile(regexps, arraysize(regexps), &v); + EXPECT_TRUE(CheckExpectedAtoms(atoms, arraysize(atoms), + "EmptyStringInStringSetBug", &v)); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/mimics_pcre_test.cc b/extern/re2/re2/testing/mimics_pcre_test.cc new file mode 100644 index 0000000000..01ab41ee38 --- /dev/null +++ b/extern/re2/re2/testing/mimics_pcre_test.cc @@ -0,0 +1,77 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "util/test.h" +#include "util/logging.h" +#include "re2/prog.h" +#include "re2/regexp.h" + +namespace re2 { + +struct PCRETest { + const char* regexp; + bool should_match; +}; + +static PCRETest tests[] = { + // Most things should behave exactly. + { "abc", true }, + { "(a|b)c", true }, + { "(a*|b)c", true }, + { "(a|b*)c", true }, + { "a(b|c)d", true }, + { "a(()|())c", true }, + { "ab*c", true }, + { "ab+c", true }, + { "a(b*|c*)d", true }, + { "\\W", true }, + { "\\W{1,2}", true }, + { "\\d", true }, + + // Check that repeated empty strings do not. + { "(a*)*", false }, + { "x(a*)*y", false }, + { "(a*)+", false }, + { "(a+)*", true }, + { "(a+)+", true }, + { "(a+)+", true }, + + // \v is the only character class that shouldn't. + { "\\b", true }, + { "\\v", false }, + { "\\d", true }, + + // The handling of ^ in multi-line mode is different, as is + // the handling of $ in single-line mode. (Both involve + // boundary cases if the string ends with \n.) + { "\\A", true }, + { "\\z", true }, + { "(?m)^", false }, + { "(?m)$", true }, + { "(?-m)^", true }, + { "(?-m)$", false }, // In PCRE, == \Z + { "(?m)\\A", true }, + { "(?m)\\z", true }, + { "(?-m)\\A", true }, + { "(?-m)\\z", true }, +}; + +TEST(MimicsPCRE, SimpleTests) { + for (size_t i = 0; i < arraysize(tests); i++) { + const PCRETest& t = tests[i]; + for (size_t j = 0; j < 2; j++) { + Regexp::ParseFlags flags = Regexp::LikePerl; + if (j == 0) + flags = flags | Regexp::Latin1; + Regexp* re = Regexp::Parse(t.regexp, flags, NULL); + ASSERT_TRUE(re != NULL) << " " << t.regexp; + ASSERT_EQ(t.should_match, re->MimicsPCRE()) + << " " << t.regexp << " " + << (j == 0 ? "latin1" : "utf"); + re->Decref(); + } + } +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/null_walker.cc b/extern/re2/re2/testing/null_walker.cc new file mode 100644 index 0000000000..77fa72389e --- /dev/null +++ b/extern/re2/re2/testing/null_walker.cc @@ -0,0 +1,46 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "util/test.h" +#include "util/logging.h" +#include "re2/regexp.h" +#include "re2/walker-inl.h" + +namespace re2 { + +// Null walker. For benchmarking the walker itself. + +class NullWalker : public Regexp::Walker { + public: + NullWalker() { } + bool PostVisit(Regexp* re, bool parent_arg, bool pre_arg, + bool* child_args, int nchild_args); + + bool ShortVisit(Regexp* re, bool a) { + // Should never be called: we use Walk not WalkExponential. + LOG(DFATAL) << "NullWalker::ShortVisit called"; + return a; + } + + private: + NullWalker(const NullWalker&) = delete; + NullWalker& operator=(const NullWalker&) = delete; +}; + +// Called after visiting re's children. child_args contains the return +// value from each of the children's PostVisits (i.e., whether each child +// can match an empty string). Returns whether this clause can match an +// empty string. +bool NullWalker::PostVisit(Regexp* re, bool parent_arg, bool pre_arg, + bool* child_args, int nchild_args) { + return false; +} + +// Returns whether re can match an empty string. +void Regexp::NullWalk() { + NullWalker w; + w.Walk(this, false); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/parse_test.cc b/extern/re2/re2/testing/parse_test.cc new file mode 100644 index 0000000000..344652690f --- /dev/null +++ b/extern/re2/re2/testing/parse_test.cc @@ -0,0 +1,508 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test parse.cc, dump.cc, and tostring.cc. + +#include + +#include "util/test.h" +#include "util/logging.h" +#include "re2/regexp.h" + +namespace re2 { + +// In the past, we used 1<<30 here and zeroed the bit later, but that +// has undefined behaviour, so now we use an internal-only flag because +// otherwise we would have to introduce a new flag value just for this. +static const Regexp::ParseFlags TestZeroFlags = Regexp::WasDollar; + +struct Test { + const char* regexp; + const char* parse; + Regexp::ParseFlags flags; +}; + +static Regexp::ParseFlags kTestFlags = Regexp::MatchNL | + Regexp::PerlX | + Regexp::PerlClasses | + Regexp::UnicodeGroups; + +static Test tests[] = { + // Base cases + { "a", "lit{a}" }, + { "a.", "cat{lit{a}dot{}}" }, + { "a.b", "cat{lit{a}dot{}lit{b}}" }, + { "ab", "str{ab}" }, + { "a.b.c", "cat{lit{a}dot{}lit{b}dot{}lit{c}}" }, + { "abc", "str{abc}" }, + { "a|^", "alt{lit{a}bol{}}" }, + { "a|b", "cc{0x61-0x62}" }, + { "(a)", "cap{lit{a}}" }, + { "(a)|b", "alt{cap{lit{a}}lit{b}}" }, + { "a*", "star{lit{a}}" }, + { "a+", "plus{lit{a}}" }, + { "a?", "que{lit{a}}" }, + { "a{2}", "rep{2,2 lit{a}}" }, + { "a{2,3}", "rep{2,3 lit{a}}" }, + { "a{2,}", "rep{2,-1 lit{a}}" }, + { "a*?", "nstar{lit{a}}" }, + { "a+?", "nplus{lit{a}}" }, + { "a??", "nque{lit{a}}" }, + { "a{2}?", "nrep{2,2 lit{a}}" }, + { "a{2,3}?", "nrep{2,3 lit{a}}" }, + { "a{2,}?", "nrep{2,-1 lit{a}}" }, + { "", "emp{}" }, + { "|", "alt{emp{}emp{}}" }, + { "|x|", "alt{emp{}lit{x}emp{}}" }, + { ".", "dot{}" }, + { "^", "bol{}" }, + { "$", "eol{}" }, + { "\\|", "lit{|}" }, + { "\\(", "lit{(}" }, + { "\\)", "lit{)}" }, + { "\\*", "lit{*}" }, + { "\\+", "lit{+}" }, + { "\\?", "lit{?}" }, + { "{", "lit{{}" }, + { "}", "lit{}}" }, + { "\\.", "lit{.}" }, + { "\\^", "lit{^}" }, + { "\\$", "lit{$}" }, + { "\\\\", "lit{\\}" }, + { "[ace]", "cc{0x61 0x63 0x65}" }, + { "[abc]", "cc{0x61-0x63}" }, + { "[a-z]", "cc{0x61-0x7a}" }, + { "[a]", "lit{a}" }, + { "\\-", "lit{-}" }, + { "-", "lit{-}" }, + { "\\_", "lit{_}" }, + + // Posix and Perl extensions + { "[[:lower:]]", "cc{0x61-0x7a}" }, + { "[a-z]", "cc{0x61-0x7a}" }, + { "[^[:lower:]]", "cc{0-0x60 0x7b-0x10ffff}" }, + { "[[:^lower:]]", "cc{0-0x60 0x7b-0x10ffff}" }, + { "(?i)[[:lower:]]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}" }, + { "(?i)[a-z]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}" }, + { "(?i)[^[:lower:]]", "cc{0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}" }, + { "(?i)[[:^lower:]]", "cc{0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}" }, + { "\\d", "cc{0x30-0x39}" }, + { "\\D", "cc{0-0x2f 0x3a-0x10ffff}" }, + { "\\s", "cc{0x9-0xa 0xc-0xd 0x20}" }, + { "\\S", "cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}" }, + { "\\w", "cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a}" }, + { "\\W", "cc{0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x10ffff}" }, + { "(?i)\\w", "cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a 0x17f 0x212a}" }, + { "(?i)\\W", "cc{0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}" }, + { "[^\\\\]", "cc{0-0x5b 0x5d-0x10ffff}" }, + { "\\C", "byte{}" }, + + // Unicode, negatives, and a double negative. + { "\\p{Braille}", "cc{0x2800-0x28ff}" }, + { "\\P{Braille}", "cc{0-0x27ff 0x2900-0x10ffff}" }, + { "\\p{^Braille}", "cc{0-0x27ff 0x2900-0x10ffff}" }, + { "\\P{^Braille}", "cc{0x2800-0x28ff}" }, + + // More interesting regular expressions. + { "a{,2}", "str{a{,2}}" }, + { "\\.\\^\\$\\\\", "str{.^$\\}" }, + { "[a-zABC]", "cc{0x41-0x43 0x61-0x7a}" }, + { "[^a]", "cc{0-0x60 0x62-0x10ffff}" }, + { "[\xce\xb1-\xce\xb5\xe2\x98\xba]", "cc{0x3b1-0x3b5 0x263a}" }, // utf-8 + { "a*{", "cat{star{lit{a}}lit{{}}" }, + + // Test precedences + { "(?:ab)*", "star{str{ab}}" }, + { "(ab)*", "star{cap{str{ab}}}" }, + { "ab|cd", "alt{str{ab}str{cd}}" }, + { "a(b|c)d", "cat{lit{a}cap{cc{0x62-0x63}}lit{d}}" }, + + // Test squashing of **, ++, ?? et cetera. + { "(?:(?:a)*)*", "star{lit{a}}" }, + { "(?:(?:a)+)+", "plus{lit{a}}" }, + { "(?:(?:a)?)?", "que{lit{a}}" }, + { "(?:(?:a)*)+", "star{lit{a}}" }, + { "(?:(?:a)*)?", "star{lit{a}}" }, + { "(?:(?:a)+)*", "star{lit{a}}" }, + { "(?:(?:a)+)?", "star{lit{a}}" }, + { "(?:(?:a)?)*", "star{lit{a}}" }, + { "(?:(?:a)?)+", "star{lit{a}}" }, + + // Test flattening. + { "(?:a)", "lit{a}" }, + { "(?:ab)(?:cd)", "str{abcd}" }, + { "(?:a|b)|(?:c|d)", "cc{0x61-0x64}" }, + { "a|c", "cc{0x61 0x63}" }, + { "a|[cd]", "cc{0x61 0x63-0x64}" }, + { "a|.", "dot{}" }, + { "[ab]|c", "cc{0x61-0x63}" }, + { "[ab]|[cd]", "cc{0x61-0x64}" }, + { "[ab]|.", "dot{}" }, + { ".|c", "dot{}" }, + { ".|[cd]", "dot{}" }, + { ".|.", "dot{}" }, + + // Test Perl quoted literals + { "\\Q+|*?{[\\E", "str{+|*?{[}" }, + { "\\Q+\\E+", "plus{lit{+}}" }, + { "\\Q\\\\E", "lit{\\}" }, + { "\\Q\\\\\\E", "str{\\\\}" }, + { "\\Qa\\E*", "star{lit{a}}" }, + { "\\Qab\\E*", "cat{lit{a}star{lit{b}}}" }, + { "\\Qabc\\E*", "cat{str{ab}star{lit{c}}}" }, + + // Test Perl \A and \z + { "(?m)^", "bol{}" }, + { "(?m)$", "eol{}" }, + { "(?-m)^", "bot{}" }, + { "(?-m)$", "eot{}" }, + { "(?m)\\A", "bot{}" }, + { "(?m)\\z", "eot{\\z}" }, + { "(?-m)\\A", "bot{}" }, + { "(?-m)\\z", "eot{\\z}" }, + + // Test named captures + { "(?Pa)", "cap{name:lit{a}}" }, + + // Case-folded literals + { "[Aa]", "litfold{a}" }, + + // Strings + { "abcde", "str{abcde}" }, + { "[Aa][Bb]cd", "cat{strfold{ab}str{cd}}" }, + + // Reported bug involving \n leaking in despite use of NeverNL. + { "[^ ]", "cc{0-0x9 0xb-0x1f 0x21-0x10ffff}", TestZeroFlags }, + { "[^ ]", "cc{0-0x9 0xb-0x1f 0x21-0x10ffff}", Regexp::FoldCase }, + { "[^ ]", "cc{0-0x9 0xb-0x1f 0x21-0x10ffff}", Regexp::NeverNL }, + { "[^ ]", "cc{0-0x9 0xb-0x1f 0x21-0x10ffff}", Regexp::NeverNL | Regexp::FoldCase }, + { "[^ \f]", "cc{0-0x9 0xb 0xd-0x1f 0x21-0x10ffff}", TestZeroFlags }, + { "[^ \f]", "cc{0-0x9 0xb 0xd-0x1f 0x21-0x10ffff}", Regexp::FoldCase }, + { "[^ \f]", "cc{0-0x9 0xb 0xd-0x1f 0x21-0x10ffff}", Regexp::NeverNL }, + { "[^ \f]", "cc{0-0x9 0xb 0xd-0x1f 0x21-0x10ffff}", Regexp::NeverNL | Regexp::FoldCase }, + { "[^ \r]", "cc{0-0x9 0xb-0xc 0xe-0x1f 0x21-0x10ffff}", TestZeroFlags }, + { "[^ \r]", "cc{0-0x9 0xb-0xc 0xe-0x1f 0x21-0x10ffff}", Regexp::FoldCase }, + { "[^ \r]", "cc{0-0x9 0xb-0xc 0xe-0x1f 0x21-0x10ffff}", Regexp::NeverNL }, + { "[^ \r]", "cc{0-0x9 0xb-0xc 0xe-0x1f 0x21-0x10ffff}", Regexp::NeverNL | Regexp::FoldCase }, + { "[^ \v]", "cc{0-0x9 0xc-0x1f 0x21-0x10ffff}", TestZeroFlags }, + { "[^ \v]", "cc{0-0x9 0xc-0x1f 0x21-0x10ffff}", Regexp::FoldCase }, + { "[^ \v]", "cc{0-0x9 0xc-0x1f 0x21-0x10ffff}", Regexp::NeverNL }, + { "[^ \v]", "cc{0-0x9 0xc-0x1f 0x21-0x10ffff}", Regexp::NeverNL | Regexp::FoldCase }, + { "[^ \t]", "cc{0-0x8 0xb-0x1f 0x21-0x10ffff}", TestZeroFlags }, + { "[^ \t]", "cc{0-0x8 0xb-0x1f 0x21-0x10ffff}", Regexp::FoldCase }, + { "[^ \t]", "cc{0-0x8 0xb-0x1f 0x21-0x10ffff}", Regexp::NeverNL }, + { "[^ \t]", "cc{0-0x8 0xb-0x1f 0x21-0x10ffff}", Regexp::NeverNL | Regexp::FoldCase }, + { "[^ \r\f\v]", "cc{0-0x9 0xe-0x1f 0x21-0x10ffff}", Regexp::NeverNL }, + { "[^ \r\f\v]", "cc{0-0x9 0xe-0x1f 0x21-0x10ffff}", Regexp::NeverNL | Regexp::FoldCase }, + { "[^ \r\f\t\v]", "cc{0-0x8 0xe-0x1f 0x21-0x10ffff}", Regexp::NeverNL }, + { "[^ \r\f\t\v]", "cc{0-0x8 0xe-0x1f 0x21-0x10ffff}", Regexp::NeverNL | Regexp::FoldCase }, + { "[^ \r\n\f\t\v]", "cc{0-0x8 0xe-0x1f 0x21-0x10ffff}", Regexp::NeverNL }, + { "[^ \r\n\f\t\v]", "cc{0-0x8 0xe-0x1f 0x21-0x10ffff}", Regexp::NeverNL | Regexp::FoldCase }, + { "[^ \r\n\f\t]", "cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}", Regexp::NeverNL }, + { "[^ \r\n\f\t]", "cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}", Regexp::NeverNL | Regexp::FoldCase }, + { "[^\t-\n\f-\r ]", "cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}", + Regexp::PerlClasses }, + { "[^\t-\n\f-\r ]", "cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}", + Regexp::PerlClasses | Regexp::FoldCase }, + { "[^\t-\n\f-\r ]", "cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}", + Regexp::PerlClasses | Regexp::NeverNL }, + { "[^\t-\n\f-\r ]", "cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}", + Regexp::PerlClasses | Regexp::NeverNL | Regexp::FoldCase }, + { "\\S", "cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}", + Regexp::PerlClasses }, + { "\\S", "cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}", + Regexp::PerlClasses | Regexp::FoldCase }, + { "\\S", "cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}", + Regexp::PerlClasses | Regexp::NeverNL }, + { "\\S", "cc{0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}", + Regexp::PerlClasses | Regexp::NeverNL | Regexp::FoldCase }, + + // Bug in Regexp::ToString() that emitted [^], which + // would (obviously) fail to parse when fed back in. + { "[\\s\\S]", "cc{0-0x10ffff}" }, +}; + +bool RegexpEqualTestingOnly(Regexp* a, Regexp* b) { + return Regexp::Equal(a, b); +} + +void TestParse(const Test* tests, int ntests, Regexp::ParseFlags flags, + const std::string& title) { + Regexp** re = new Regexp*[ntests]; + for (int i = 0; i < ntests; i++) { + RegexpStatus status; + Regexp::ParseFlags f = flags; + if (tests[i].flags != 0) { + f = tests[i].flags & ~TestZeroFlags; + } + re[i] = Regexp::Parse(tests[i].regexp, f, &status); + ASSERT_TRUE(re[i] != NULL) + << " " << tests[i].regexp << " " << status.Text(); + std::string s = re[i]->Dump(); + EXPECT_EQ(std::string(tests[i].parse), s) + << "Regexp: " << tests[i].regexp + << "\nparse: " << std::string(tests[i].parse) + << " s: " << s << " flag=" << f; + } + + for (int i = 0; i < ntests; i++) { + for (int j = 0; j < ntests; j++) { + EXPECT_EQ(std::string(tests[i].parse) == std::string(tests[j].parse), + RegexpEqualTestingOnly(re[i], re[j])) + << "Regexp: " << tests[i].regexp << " " << tests[j].regexp; + } + } + + for (int i = 0; i < ntests; i++) + re[i]->Decref(); + delete[] re; +} + +// Test that regexps parse to expected structures. +TEST(TestParse, SimpleRegexps) { + TestParse(tests, arraysize(tests), kTestFlags, "simple"); +} + +Test foldcase_tests[] = { + { "AbCdE", "strfold{abcde}" }, + { "[Aa]", "litfold{a}" }, + { "a", "litfold{a}" }, + + // 0x17F is an old English long s (looks like an f) and folds to s. + // 0x212A is the Kelvin symbol and folds to k. + { "A[F-g]", "cat{litfold{a}cc{0x41-0x7a 0x17f 0x212a}}" }, // [Aa][A-z...] + { "[[:upper:]]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}" }, + { "[[:lower:]]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}" }, +}; + +// Test that parsing with FoldCase works. +TEST(TestParse, FoldCase) { + TestParse(foldcase_tests, arraysize(foldcase_tests), Regexp::FoldCase, "foldcase"); +} + +Test literal_tests[] = { + { "(|)^$.[*+?]{5,10},\\", "str{(|)^$.[*+?]{5,10},\\}" }, +}; + +// Test that parsing with Literal works. +TEST(TestParse, Literal) { + TestParse(literal_tests, arraysize(literal_tests), Regexp::Literal, "literal"); +} + +Test matchnl_tests[] = { + { ".", "dot{}" }, + { "\n", "lit{\n}" }, + { "[^a]", "cc{0-0x60 0x62-0x10ffff}" }, + { "[a\\n]", "cc{0xa 0x61}" }, +}; + +// Test that parsing with MatchNL works. +// (Also tested above during simple cases.) +TEST(TestParse, MatchNL) { + TestParse(matchnl_tests, arraysize(matchnl_tests), Regexp::MatchNL, "with MatchNL"); +} + +Test nomatchnl_tests[] = { + { ".", "cc{0-0x9 0xb-0x10ffff}" }, + { "\n", "lit{\n}" }, + { "[^a]", "cc{0-0x9 0xb-0x60 0x62-0x10ffff}" }, + { "[a\\n]", "cc{0xa 0x61}" }, +}; + +// Test that parsing without MatchNL works. +TEST(TestParse, NoMatchNL) { + TestParse(nomatchnl_tests, arraysize(nomatchnl_tests), Regexp::NoParseFlags, "without MatchNL"); +} + +Test prefix_tests[] = { + { "abc|abd", "cat{str{ab}cc{0x63-0x64}}" }, + { "a(?:b)c|abd", "cat{str{ab}cc{0x63-0x64}}" }, + { "abc|abd|aef|bcx|bcy", + "alt{cat{lit{a}alt{cat{lit{b}cc{0x63-0x64}}str{ef}}}" + "cat{str{bc}cc{0x78-0x79}}}" }, + { "abc|x|abd", "alt{str{abc}lit{x}str{abd}}" }, + { "(?i)abc|ABD", "cat{strfold{ab}cc{0x43-0x44 0x63-0x64}}" }, + { "[ab]c|[ab]d", "cat{cc{0x61-0x62}cc{0x63-0x64}}" }, + { ".c|.d", "cat{cc{0-0x9 0xb-0x10ffff}cc{0x63-0x64}}" }, + { "\\Cc|\\Cd", "cat{byte{}cc{0x63-0x64}}" }, + { "x{2}|x{2}[0-9]", + "cat{rep{2,2 lit{x}}alt{emp{}cc{0x30-0x39}}}" }, + { "x{2}y|x{2}[0-9]y", + "cat{rep{2,2 lit{x}}alt{lit{y}cat{cc{0x30-0x39}lit{y}}}}" }, + { "n|r|rs", + "alt{lit{n}cat{lit{r}alt{emp{}lit{s}}}}" }, + { "n|rs|r", + "alt{lit{n}cat{lit{r}alt{lit{s}emp{}}}}" }, + { "r|rs|n", + "alt{cat{lit{r}alt{emp{}lit{s}}}lit{n}}" }, + { "rs|r|n", + "alt{cat{lit{r}alt{lit{s}emp{}}}lit{n}}" }, + { "a\\C*?c|a\\C*?b", + "cat{lit{a}alt{cat{nstar{byte{}}lit{c}}cat{nstar{byte{}}lit{b}}}}" }, + { "^/a/bc|^/a/de", + "cat{bol{}cat{str{/a/}alt{str{bc}str{de}}}}" }, + // In the past, factoring was limited to kFactorAlternationMaxDepth (8). + { "a|aa|aaa|aaaa|aaaaa|aaaaaa|aaaaaaa|aaaaaaaa|aaaaaaaaa|aaaaaaaaaa", + "cat{lit{a}alt{emp{}" "cat{lit{a}alt{emp{}" "cat{lit{a}alt{emp{}" + "cat{lit{a}alt{emp{}" "cat{lit{a}alt{emp{}" "cat{lit{a}alt{emp{}" + "cat{lit{a}alt{emp{}" "cat{lit{a}alt{emp{}" "cat{lit{a}alt{emp{}" + "lit{a}}}}}}}}}}}}}}}}}}}" }, + { "a|aardvark|aardvarks|abaci|aback|abacus|abacuses|abaft|abalone|abalones", + "cat{lit{a}alt{emp{}cat{str{ardvark}alt{emp{}lit{s}}}" + "cat{str{ba}alt{cat{lit{c}alt{cc{0x69 0x6b}cat{str{us}alt{emp{}str{es}}}}}" + "str{ft}cat{str{lone}alt{emp{}lit{s}}}}}}}" }, +}; + +// Test that prefix factoring works. +TEST(TestParse, Prefix) { + TestParse(prefix_tests, arraysize(prefix_tests), Regexp::PerlX, "prefix"); +} + +Test nested_tests[] = { + { "((((((((((x{2}){2}){2}){2}){2}){2}){2}){2}){2}))", + "cap{cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 lit{x}}}}}}}}}}}}}}}}}}}}" }, + { "((((((((((x{1}){2}){2}){2}){2}){2}){2}){2}){2}){2})", + "cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{1,1 lit{x}}}}}}}}}}}}}}}}}}}}}" }, + { "((((((((((x{0}){2}){2}){2}){2}){2}){2}){2}){2}){2})", + "cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 cap{rep{0,0 lit{x}}}}}}}}}}}}}}}}}}}}}" }, + { "((((((x{2}){2}){2}){5}){5}){5})", + "cap{rep{5,5 cap{rep{5,5 cap{rep{5,5 cap{rep{2,2 cap{rep{2,2 cap{rep{2,2 lit{x}}}}}}}}}}}}}" }, +}; + +// Test that nested repetition works. +TEST(TestParse, Nested) { + TestParse(nested_tests, arraysize(nested_tests), Regexp::PerlX, "nested"); +} + +// Invalid regular expressions +const char* badtests[] = { + "(", + ")", + "(a", + "(a|b|", + "(a|b", + "[a-z", + "([a-z)", + "x{1001}", + "\xff", // Invalid UTF-8 + "[\xff]", + "[\\\xff]", + "\\\xff", + "(?Pa", + "(?P", + "(?Pa)", + "(?P<>a)", + "[a-Z]", + "(?i)[a-Z]", + "a{100000}", + "a{100000,}", + "((((((((((x{2}){2}){2}){2}){2}){2}){2}){2}){2}){2})", + "(((x{7}){11}){13})", + "\\Q\\E*", +}; + +// Valid in Perl, bad in POSIX +const char* only_perl[] = { + "[a-b-c]", + "\\Qabc\\E", + "\\Q*+?{[\\E", + "\\Q\\\\E", + "\\Q\\\\\\E", + "\\Q\\\\\\\\E", + "\\Q\\\\\\\\\\E", + "(?:a)", + "(?Pa)", +}; + +// Valid in POSIX, bad in Perl. +const char* only_posix[] = { + "a++", + "a**", + "a?*", + "a+*", + "a{1}*", +}; + +// Test that parser rejects bad regexps. +TEST(TestParse, InvalidRegexps) { + for (size_t i = 0; i < arraysize(badtests); i++) { + ASSERT_TRUE(Regexp::Parse(badtests[i], Regexp::PerlX, NULL) == NULL) + << " " << badtests[i]; + ASSERT_TRUE(Regexp::Parse(badtests[i], Regexp::NoParseFlags, NULL) == NULL) + << " " << badtests[i]; + } + for (size_t i = 0; i < arraysize(only_posix); i++) { + ASSERT_TRUE(Regexp::Parse(only_posix[i], Regexp::PerlX, NULL) == NULL) + << " " << only_posix[i]; + Regexp* re = Regexp::Parse(only_posix[i], Regexp::NoParseFlags, NULL); + ASSERT_TRUE(re != NULL) << " " << only_posix[i]; + re->Decref(); + } + for (size_t i = 0; i < arraysize(only_perl); i++) { + ASSERT_TRUE(Regexp::Parse(only_perl[i], Regexp::NoParseFlags, NULL) == NULL) + << " " << only_perl[i]; + Regexp* re = Regexp::Parse(only_perl[i], Regexp::PerlX, NULL); + ASSERT_TRUE(re != NULL) << " " << only_perl[i]; + re->Decref(); + } +} + +// Test that ToString produces original regexp or equivalent one. +TEST(TestToString, EquivalentParse) { + for (size_t i = 0; i < arraysize(tests); i++) { + RegexpStatus status; + Regexp::ParseFlags f = kTestFlags; + if (tests[i].flags != 0) { + f = tests[i].flags & ~TestZeroFlags; + } + Regexp* re = Regexp::Parse(tests[i].regexp, f, &status); + ASSERT_TRUE(re != NULL) << " " << tests[i].regexp << " " << status.Text(); + std::string s = re->Dump(); + EXPECT_EQ(std::string(tests[i].parse), s) + << "Regexp: " << tests[i].regexp + << "\nparse: " << std::string(tests[i].parse) + << " s: " << s << " flag=" << f; + std::string t = re->ToString(); + if (t != tests[i].regexp) { + // If ToString didn't return the original regexp, + // it must have found one with fewer parens. + // Unfortunately we can't check the length here, because + // ToString produces "\\{" for a literal brace, + // but "{" is a shorter equivalent. + // ASSERT_LT(t.size(), strlen(tests[i].regexp)) + // << " t=" << t << " regexp=" << tests[i].regexp; + + // Test that if we parse the new regexp we get the same structure. + Regexp* nre = Regexp::Parse(t, Regexp::MatchNL | Regexp::PerlX, &status); + ASSERT_TRUE(nre != NULL) << " reparse " << t << " " << status.Text(); + std::string ss = nre->Dump(); + std::string tt = nre->ToString(); + if (s != ss || t != tt) + LOG(INFO) << "ToString(" << tests[i].regexp << ") = " << t; + EXPECT_EQ(s, ss); + EXPECT_EQ(t, tt); + nre->Decref(); + } + re->Decref(); + } +} + +// Test that capture error args are correct. +TEST(NamedCaptures, ErrorArgs) { + RegexpStatus status; + Regexp* re; + + re = Regexp::Parse("test(?Pz)", Regexp::LikePerl, &status); + EXPECT_TRUE(re == NULL); + EXPECT_EQ(status.code(), kRegexpBadNamedCapture); + EXPECT_EQ(status.error_arg(), "(?P"); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/possible_match_test.cc b/extern/re2/re2/testing/possible_match_test.cc new file mode 100644 index 0000000000..0ec90ae1e5 --- /dev/null +++ b/extern/re2/re2/testing/possible_match_test.cc @@ -0,0 +1,247 @@ +// Copyright 2006-2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include +#include + +#include "util/test.h" +#include "util/logging.h" +#include "util/strutil.h" +#include "re2/prog.h" +#include "re2/re2.h" +#include "re2/regexp.h" +#include "re2/testing/exhaustive_tester.h" +#include "re2/testing/regexp_generator.h" +#include "re2/testing/string_generator.h" + +namespace re2 { + +// Test that C++ strings are compared as uint8s, not int8s. +// PossibleMatchRange doesn't depend on this, but callers probably will. +TEST(CplusplusStrings, EightBit) { + std::string s = "\x70"; + std::string t = "\xA0"; + EXPECT_LT(s, t); +} + +struct PrefixTest { + const char* regexp; + int maxlen; + const char* min; + const char* max; +}; + +static PrefixTest tests[] = { + { "", 10, "", "", }, + { "Abcdef", 10, "Abcdef", "Abcdef" }, + { "abc(def|ghi)", 10, "abcdef", "abcghi" }, + { "a+hello", 10, "aa", "ahello" }, + { "a*hello", 10, "a", "hello" }, + { "def|abc", 10, "abc", "def" }, + { "a(b)(c)[d]", 10, "abcd", "abcd" }, + { "ab(cab|cat)", 10, "abcab", "abcat" }, + { "ab(cab|ca)x", 10, "abcabx", "abcax" }, + { "(ab|x)(c|de)", 10, "abc", "xde" }, + { "(ab|x)?(c|z)?", 10, "", "z" }, + { "[^\\s\\S]", 10, "", "" }, + { "(abc)+", 5, "abc", "abcac" }, + { "(abc)+", 2, "ab", "ac" }, + { "(abc)+", 1, "a", "b" }, + { "[a\xC3\xA1]", 4, "a", "\xC3\xA1" }, + { "a*", 10, "", "ab" }, + + { "(?i)Abcdef", 10, "ABCDEF", "abcdef" }, + { "(?i)abc(def|ghi)", 10, "ABCDEF", "abcghi" }, + { "(?i)a+hello", 10, "AA", "ahello" }, + { "(?i)a*hello", 10, "A", "hello" }, + { "(?i)def|abc", 10, "ABC", "def" }, + { "(?i)a(b)(c)[d]", 10, "ABCD", "abcd" }, + { "(?i)ab(cab|cat)", 10, "ABCAB", "abcat" }, + { "(?i)ab(cab|ca)x", 10, "ABCABX", "abcax" }, + { "(?i)(ab|x)(c|de)", 10, "ABC", "xde" }, + { "(?i)(ab|x)?(c|z)?", 10, "", "z" }, + { "(?i)[^\\s\\S]", 10, "", "" }, + { "(?i)(abc)+", 5, "ABC", "abcac" }, + { "(?i)(abc)+", 2, "AB", "ac" }, + { "(?i)(abc)+", 1, "A", "b" }, + { "(?i)[a\xC3\xA1]", 4, "A", "\xC3\xA1" }, + { "(?i)a*", 10, "", "ab" }, + { "(?i)A*", 10, "", "ab" }, + + { "\\AAbcdef", 10, "Abcdef", "Abcdef" }, + { "\\Aabc(def|ghi)", 10, "abcdef", "abcghi" }, + { "\\Aa+hello", 10, "aa", "ahello" }, + { "\\Aa*hello", 10, "a", "hello" }, + { "\\Adef|abc", 10, "abc", "def" }, + { "\\Aa(b)(c)[d]", 10, "abcd", "abcd" }, + { "\\Aab(cab|cat)", 10, "abcab", "abcat" }, + { "\\Aab(cab|ca)x", 10, "abcabx", "abcax" }, + { "\\A(ab|x)(c|de)", 10, "abc", "xde" }, + { "\\A(ab|x)?(c|z)?", 10, "", "z" }, + { "\\A[^\\s\\S]", 10, "", "" }, + { "\\A(abc)+", 5, "abc", "abcac" }, + { "\\A(abc)+", 2, "ab", "ac" }, + { "\\A(abc)+", 1, "a", "b" }, + { "\\A[a\xC3\xA1]", 4, "a", "\xC3\xA1" }, + { "\\Aa*", 10, "", "ab" }, + + { "(?i)\\AAbcdef", 10, "ABCDEF", "abcdef" }, + { "(?i)\\Aabc(def|ghi)", 10, "ABCDEF", "abcghi" }, + { "(?i)\\Aa+hello", 10, "AA", "ahello" }, + { "(?i)\\Aa*hello", 10, "A", "hello" }, + { "(?i)\\Adef|abc", 10, "ABC", "def" }, + { "(?i)\\Aa(b)(c)[d]", 10, "ABCD", "abcd" }, + { "(?i)\\Aab(cab|cat)", 10, "ABCAB", "abcat" }, + { "(?i)\\Aab(cab|ca)x", 10, "ABCABX", "abcax" }, + { "(?i)\\A(ab|x)(c|de)", 10, "ABC", "xde" }, + { "(?i)\\A(ab|x)?(c|z)?", 10, "", "z" }, + { "(?i)\\A[^\\s\\S]", 10, "", "" }, + { "(?i)\\A(abc)+", 5, "ABC", "abcac" }, + { "(?i)\\A(abc)+", 2, "AB", "ac" }, + { "(?i)\\A(abc)+", 1, "A", "b" }, + { "(?i)\\A[a\xC3\xA1]", 4, "A", "\xC3\xA1" }, + { "(?i)\\Aa*", 10, "", "ab" }, + { "(?i)\\AA*", 10, "", "ab" }, +}; + +TEST(PossibleMatchRange, HandWritten) { + for (size_t i = 0; i < arraysize(tests); i++) { + for (size_t j = 0; j < 2; j++) { + const PrefixTest& t = tests[i]; + std::string min, max; + if (j == 0) { + LOG(INFO) << "Checking regexp=" << CEscape(t.regexp); + Regexp* re = Regexp::Parse(t.regexp, Regexp::LikePerl, NULL); + ASSERT_TRUE(re != NULL); + Prog* prog = re->CompileToProg(0); + ASSERT_TRUE(prog != NULL); + ASSERT_TRUE(prog->PossibleMatchRange(&min, &max, t.maxlen)) + << " " << t.regexp; + delete prog; + re->Decref(); + } else { + ASSERT_TRUE(RE2(t.regexp).PossibleMatchRange(&min, &max, t.maxlen)); + } + EXPECT_EQ(t.min, min) << t.regexp; + EXPECT_EQ(t.max, max) << t.regexp; + } + } +} + +// Test cases where PossibleMatchRange should return false. +TEST(PossibleMatchRange, Failures) { + std::string min, max; + + // Fails because no room to write max. + EXPECT_FALSE(RE2("abc").PossibleMatchRange(&min, &max, 0)); + + // Fails because there is no max -- any non-empty string matches + // or begins a match. Have to use Latin-1 input, because there + // are no valid UTF-8 strings beginning with byte 0xFF. + EXPECT_FALSE(RE2("[\\s\\S]+", RE2::Latin1). + PossibleMatchRange(&min, &max, 10)) + << "min=" << CEscape(min) << ", max=" << CEscape(max); + EXPECT_FALSE(RE2("[\\0-\xFF]+", RE2::Latin1). + PossibleMatchRange(&min, &max, 10)) + << "min=" << CEscape(min) << ", max=" << CEscape(max); + EXPECT_FALSE(RE2(".+hello", RE2::Latin1). + PossibleMatchRange(&min, &max, 10)) + << "min=" << CEscape(min) << ", max=" << CEscape(max); + EXPECT_FALSE(RE2(".*hello", RE2::Latin1). + PossibleMatchRange(&min, &max, 10)) + << "min=" << CEscape(min) << ", max=" << CEscape(max); + EXPECT_FALSE(RE2(".*", RE2::Latin1). + PossibleMatchRange(&min, &max, 10)) + << "min=" << CEscape(min) << ", max=" << CEscape(max); + EXPECT_FALSE(RE2("\\C*"). + PossibleMatchRange(&min, &max, 10)) + << "min=" << CEscape(min) << ", max=" << CEscape(max); + + // Fails because it's a malformed regexp. + EXPECT_FALSE(RE2("*hello").PossibleMatchRange(&min, &max, 10)) + << "min=" << CEscape(min) << ", max=" << CEscape(max); +} + +// Exhaustive test: generate all regexps within parameters, +// then generate all strings of a given length over a given alphabet, +// then check that the prefix information agrees with whether +// the regexp matches each of the strings. +class PossibleMatchTester : public RegexpGenerator { + public: + PossibleMatchTester(int maxatoms, + int maxops, + const std::vector& alphabet, + const std::vector& ops, + int maxstrlen, + const std::vector& stralphabet) + : RegexpGenerator(maxatoms, maxops, alphabet, ops), + strgen_(maxstrlen, stralphabet), + regexps_(0), tests_(0) { } + + int regexps() { return regexps_; } + int tests() { return tests_; } + + // Needed for RegexpGenerator interface. + void HandleRegexp(const std::string& regexp); + + private: + StringGenerator strgen_; + + int regexps_; // Number of HandleRegexp calls + int tests_; // Number of regexp tests. + + PossibleMatchTester(const PossibleMatchTester&) = delete; + PossibleMatchTester& operator=(const PossibleMatchTester&) = delete; +}; + +// Processes a single generated regexp. +// Checks that all accepted strings agree with the prefix range. +void PossibleMatchTester::HandleRegexp(const std::string& regexp) { + regexps_++; + + VLOG(3) << CEscape(regexp); + + RE2 re(regexp, RE2::Latin1); + ASSERT_EQ(re.error(), ""); + + std::string min, max; + if(!re.PossibleMatchRange(&min, &max, 10)) { + // There's no good max for "\\C*". Can't use strcmp + // because sometimes it gets embedded in more + // complicated expressions. + if(strstr(regexp.c_str(), "\\C*")) + return; + LOG(QFATAL) << "PossibleMatchRange failed on: " << CEscape(regexp); + } + + strgen_.Reset(); + while (strgen_.HasNext()) { + const StringPiece& s = strgen_.Next(); + tests_++; + if (!RE2::FullMatch(s, re)) + continue; + ASSERT_GE(s, min) << " regexp: " << regexp << " max: " << max; + ASSERT_LE(s, max) << " regexp: " << regexp << " min: " << min; + } +} + +TEST(PossibleMatchRange, Exhaustive) { + int natom = 3; + int noperator = 3; + int stringlen = 5; + if (RE2_DEBUG_MODE) { + natom = 2; + noperator = 3; + stringlen = 3; + } + PossibleMatchTester t(natom, noperator, Split(" ", "a b [0-9]"), + RegexpGenerator::EgrepOps(), + stringlen, Explode("ab4")); + t.Generate(); + LOG(INFO) << t.regexps() << " regexps, " + << t.tests() << " tests"; +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/random_test.cc b/extern/re2/re2/testing/random_test.cc new file mode 100644 index 0000000000..c0b1fe5b35 --- /dev/null +++ b/extern/re2/re2/testing/random_test.cc @@ -0,0 +1,99 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Random testing of regular expression matching. + +#include +#include +#include + +#include "util/test.h" +#include "re2/testing/exhaustive_tester.h" + +DEFINE_int32(regexpseed, 404, "Random regexp seed."); +DEFINE_int32(regexpcount, 100, "How many random regexps to generate."); +DEFINE_int32(stringseed, 200, "Random string seed."); +DEFINE_int32(stringcount, 100, "How many random strings to generate."); + +namespace re2 { + +// Runs a random test on the given parameters. +// (Always uses the same random seeds for reproducibility. +// Can give different seeds on command line.) +static void RandomTest(int maxatoms, int maxops, + const std::vector& alphabet, + const std::vector& ops, + int maxstrlen, + const std::vector& stralphabet, + const std::string& wrapper) { + // Limit to smaller test cases in debug mode, + // because everything is so much slower. + if (RE2_DEBUG_MODE) { + maxatoms--; + maxops--; + maxstrlen /= 2; + } + + ExhaustiveTester t(maxatoms, maxops, alphabet, ops, + maxstrlen, stralphabet, wrapper, ""); + t.RandomStrings(FLAGS_stringseed, FLAGS_stringcount); + t.GenerateRandom(FLAGS_regexpseed, FLAGS_regexpcount); + printf("%d regexps, %d tests, %d failures [%d/%d str]\n", + t.regexps(), t.tests(), t.failures(), maxstrlen, (int)stralphabet.size()); + EXPECT_EQ(0, t.failures()); +} + +// Tests random small regexps involving literals and egrep operators. +TEST(Random, SmallEgrepLiterals) { + RandomTest(5, 5, Explode("abc."), RegexpGenerator::EgrepOps(), + 15, Explode("abc"), + ""); +} + +// Tests random bigger regexps involving literals and egrep operators. +TEST(Random, BigEgrepLiterals) { + RandomTest(10, 10, Explode("abc."), RegexpGenerator::EgrepOps(), + 15, Explode("abc"), + ""); +} + +// Tests random small regexps involving literals, capturing parens, +// and egrep operators. +TEST(Random, SmallEgrepCaptures) { + RandomTest(5, 5, Split(" ", "a (b) ."), RegexpGenerator::EgrepOps(), + 15, Explode("abc"), + ""); +} + +// Tests random bigger regexps involving literals, capturing parens, +// and egrep operators. +TEST(Random, BigEgrepCaptures) { + RandomTest(10, 10, Split(" ", "a (b) ."), RegexpGenerator::EgrepOps(), + 15, Explode("abc"), + ""); +} + +// Tests random large complicated expressions, using all the possible +// operators, some literals, some parenthesized literals, and predefined +// character classes like \d. (Adding larger character classes would +// make for too many possibilities.) +TEST(Random, Complicated) { + std::vector ops = Split(" ", + "%s%s %s|%s %s* %s*? %s+ %s+? %s? %s?? " + "%s{0} %s{0,} %s{1} %s{1,} %s{0,1} %s{0,2} %s{1,2} " + "%s{2} %s{2,} %s{3,4} %s{4,5}"); + + // Use (?:\b) and (?:\B) instead of \b and \B, + // because PCRE rejects \b* but accepts (?:\b)*. + // Ditto ^ and $. + std::vector atoms = Split(" ", + ". (?:^) (?:$) \\a \\f \\n \\r \\t \\v " + "\\d \\D \\s \\S \\w \\W (?:\\b) (?:\\B) " + "a (a) b c - \\\\"); + std::vector alphabet = Explode("abc123\001\002\003\t\r\n\v\f\a"); + RandomTest(10, 10, atoms, ops, 20, alphabet, ""); +} + +} // namespace re2 + diff --git a/extern/re2/re2/testing/re2_arg_test.cc b/extern/re2/re2/testing/re2_arg_test.cc new file mode 100644 index 0000000000..7a38de7c2f --- /dev/null +++ b/extern/re2/re2/testing/re2_arg_test.cc @@ -0,0 +1,135 @@ +// Copyright 2005 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This tests to make sure numbers are parsed from strings +// correctly. +// Todo: Expand the test to validate strings parsed to the other types +// supported by RE2::Arg class + +#include +#include + +#include "util/test.h" +#include "re2/re2.h" + +namespace re2 { + +struct SuccessTable { + const char * value_string; + int64_t value; + bool success[6]; +}; + +// Test boundary cases for different integral sizes. +// Specifically I want to make sure that values outside the boundries +// of an integral type will fail and that negative numbers will fail +// for unsigned types. The following table contains the boundaries for +// the various integral types and has entries for whether or not each +// type can contain the given value. +const SuccessTable kSuccessTable[] = { +// string integer value i16 u16 i32 u32 i64 u64 +// 0 to 2^7-1 +{ "0", 0, { true, true, true, true, true, true }}, +{ "127", 127, { true, true, true, true, true, true }}, + +// -1 to -2^7 +{ "-1", -1, { true, false, true, false, true, false }}, +{ "-128", -128, { true, false, true, false, true, false }}, + +// 2^7 to 2^8-1 +{ "128", 128, { true, true, true, true, true, true }}, +{ "255", 255, { true, true, true, true, true, true }}, + +// 2^8 to 2^15-1 +{ "256", 256, { true, true, true, true, true, true }}, +{ "32767", 32767, { true, true, true, true, true, true }}, + +// -2^7-1 to -2^15 +{ "-129", -129, { true, false, true, false, true, false }}, +{ "-32768", -32768, { true, false, true, false, true, false }}, + +// 2^15 to 2^16-1 +{ "32768", 32768, { false, true, true, true, true, true }}, +{ "65535", 65535, { false, true, true, true, true, true }}, + +// 2^16 to 2^31-1 +{ "65536", 65536, { false, false, true, true, true, true }}, +{ "2147483647", 2147483647, { false, false, true, true, true, true }}, + +// -2^15-1 to -2^31 +{ "-32769", -32769, { false, false, true, false, true, false }}, +{ "-2147483648", static_cast(0xFFFFFFFF80000000LL), + { false, false, true, false, true, false }}, + +// 2^31 to 2^32-1 +{ "2147483648", 2147483648U, { false, false, false, true, true, true }}, +{ "4294967295", 4294967295U, { false, false, false, true, true, true }}, + +// 2^32 to 2^63-1 +{ "4294967296", 4294967296LL, { false, false, false, false, true, true }}, +{ "9223372036854775807", + 9223372036854775807LL, { false, false, false, false, true, true }}, + +// -2^31-1 to -2^63 +{ "-2147483649", -2147483649LL, { false, false, false, false, true, false }}, +{ "-9223372036854775808", static_cast(0x8000000000000000LL), + { false, false, false, false, true, false }}, + +// 2^63 to 2^64-1 +{ "9223372036854775808", static_cast(9223372036854775808ULL), + { false, false, false, false, false, true }}, +{ "18446744073709551615", static_cast(18446744073709551615ULL), + { false, false, false, false, false, true }}, + +// >= 2^64 +{ "18446744073709551616", 0, { false, false, false, false, false, false }}, +}; + +const int kNumStrings = arraysize(kSuccessTable); + +// It's ugly to use a macro, but we apparently can't use the EXPECT_EQ +// macro outside of a TEST block and this seems to be the only way to +// avoid code duplication. I can also pull off a couple nice tricks +// using concatenation for the type I'm checking against. +#define PARSE_FOR_TYPE(type, column) { \ + type r; \ + for (int i = 0; i < kNumStrings; ++i) { \ + RE2::Arg arg(&r); \ + const char* const p = kSuccessTable[i].value_string; \ + bool retval = arg.Parse(p, strlen(p)); \ + bool success = kSuccessTable[i].success[column]; \ + EXPECT_EQ(retval, success) \ + << "Parsing '" << p << "' for type " #type " should return " \ + << success; \ + if (success) { \ + EXPECT_EQ(r, (type)kSuccessTable[i].value); \ + } \ + } \ +} + +TEST(RE2ArgTest, Int16Test) { + PARSE_FOR_TYPE(int16_t, 0); +} + +TEST(RE2ArgTest, Uint16Test) { + PARSE_FOR_TYPE(uint16_t, 1); +} + +TEST(RE2ArgTest, Int32Test) { + PARSE_FOR_TYPE(int32_t, 2); +} + +TEST(RE2ArgTest, Uint32Test) { + PARSE_FOR_TYPE(uint32_t, 3); +} + +TEST(RE2ArgTest, Int64Test) { + PARSE_FOR_TYPE(int64_t, 4); +} + +TEST(RE2ArgTest, Uint64Test) { + PARSE_FOR_TYPE(uint64_t, 5); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/re2_test.cc b/extern/re2/re2/testing/re2_test.cc new file mode 100644 index 0000000000..2f4b90cddd --- /dev/null +++ b/extern/re2/re2/testing/re2_test.cc @@ -0,0 +1,1631 @@ +// -*- coding: utf-8 -*- +// Copyright 2002-2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TODO: Test extractions for PartialMatch/Consume + +#include +#include +#include +#include +#include +#include +#include +#if !defined(_MSC_VER) && !defined(__CYGWIN__) && !defined(__MINGW32__) +#include +#include /* for sysconf */ +#endif + +#include "util/test.h" +#include "util/logging.h" +#include "util/strutil.h" +#include "re2/re2.h" +#include "re2/regexp.h" + +namespace re2 { + +TEST(RE2, HexTests) { +#define ASSERT_HEX(type, value) \ + do { \ + type v; \ + ASSERT_TRUE( \ + RE2::FullMatch(#value, "([0-9a-fA-F]+)[uUlL]*", RE2::Hex(&v))); \ + ASSERT_EQ(v, 0x##value); \ + ASSERT_TRUE(RE2::FullMatch("0x" #value, "([0-9a-fA-FxX]+)[uUlL]*", \ + RE2::CRadix(&v))); \ + ASSERT_EQ(v, 0x##value); \ + } while (0) + + ASSERT_HEX(short, 2bad); + ASSERT_HEX(unsigned short, 2badU); + ASSERT_HEX(int, dead); + ASSERT_HEX(unsigned int, deadU); + ASSERT_HEX(long, 7eadbeefL); + ASSERT_HEX(unsigned long, deadbeefUL); + ASSERT_HEX(long long, 12345678deadbeefLL); + ASSERT_HEX(unsigned long long, cafebabedeadbeefULL); + +#undef ASSERT_HEX +} + +TEST(RE2, OctalTests) { +#define ASSERT_OCTAL(type, value) \ + do { \ + type v; \ + ASSERT_TRUE(RE2::FullMatch(#value, "([0-7]+)[uUlL]*", RE2::Octal(&v))); \ + ASSERT_EQ(v, 0##value); \ + ASSERT_TRUE(RE2::FullMatch("0" #value, "([0-9a-fA-FxX]+)[uUlL]*", \ + RE2::CRadix(&v))); \ + ASSERT_EQ(v, 0##value); \ + } while (0) + + ASSERT_OCTAL(short, 77777); + ASSERT_OCTAL(unsigned short, 177777U); + ASSERT_OCTAL(int, 17777777777); + ASSERT_OCTAL(unsigned int, 37777777777U); + ASSERT_OCTAL(long, 17777777777L); + ASSERT_OCTAL(unsigned long, 37777777777UL); + ASSERT_OCTAL(long long, 777777777777777777777LL); + ASSERT_OCTAL(unsigned long long, 1777777777777777777777ULL); + +#undef ASSERT_OCTAL +} + +TEST(RE2, DecimalTests) { +#define ASSERT_DECIMAL(type, value) \ + do { \ + type v; \ + ASSERT_TRUE(RE2::FullMatch(#value, "(-?[0-9]+)[uUlL]*", &v)); \ + ASSERT_EQ(v, value); \ + ASSERT_TRUE( \ + RE2::FullMatch(#value, "(-?[0-9a-fA-FxX]+)[uUlL]*", RE2::CRadix(&v))); \ + ASSERT_EQ(v, value); \ + } while (0) + + ASSERT_DECIMAL(short, -1); + ASSERT_DECIMAL(unsigned short, 9999); + ASSERT_DECIMAL(int, -1000); + ASSERT_DECIMAL(unsigned int, 12345U); + ASSERT_DECIMAL(long, -10000000L); + ASSERT_DECIMAL(unsigned long, 3083324652U); + ASSERT_DECIMAL(long long, -100000000000000LL); + ASSERT_DECIMAL(unsigned long long, 1234567890987654321ULL); + +#undef ASSERT_DECIMAL +} + +TEST(RE2, Replace) { + struct ReplaceTest { + const char *regexp; + const char *rewrite; + const char *original; + const char *single; + const char *global; + int greplace_count; + }; + static const ReplaceTest tests[] = { + { "(qu|[b-df-hj-np-tv-z]*)([a-z]+)", + "\\2\\1ay", + "the quick brown fox jumps over the lazy dogs.", + "ethay quick brown fox jumps over the lazy dogs.", + "ethay ickquay ownbray oxfay umpsjay overay ethay azylay ogsday.", + 9 }, + { "\\w+", + "\\0-NOSPAM", + "abcd.efghi@google.com", + "abcd-NOSPAM.efghi@google.com", + "abcd-NOSPAM.efghi-NOSPAM@google-NOSPAM.com-NOSPAM", + 4 }, + { "^", + "(START)", + "foo", + "(START)foo", + "(START)foo", + 1 }, + { "^", + "(START)", + "", + "(START)", + "(START)", + 1 }, + { "$", + "(END)", + "", + "(END)", + "(END)", + 1 }, + { "b", + "bb", + "ababababab", + "abbabababab", + "abbabbabbabbabb", + 5 }, + { "b", + "bb", + "bbbbbb", + "bbbbbbb", + "bbbbbbbbbbbb", + 6 }, + { "b+", + "bb", + "bbbbbb", + "bb", + "bb", + 1 }, + { "b*", + "bb", + "bbbbbb", + "bb", + "bb", + 1 }, + { "b*", + "bb", + "aaaaa", + "bbaaaaa", + "bbabbabbabbabbabb", + 6 }, + // Check newline handling + { "a.*a", + "(\\0)", + "aba\naba", + "(aba)\naba", + "(aba)\n(aba)", + 2 }, + { "", NULL, NULL, NULL, NULL, 0 } + }; + + for (const ReplaceTest* t = tests; t->original != NULL; t++) { + std::string one(t->original); + ASSERT_TRUE(RE2::Replace(&one, t->regexp, t->rewrite)); + ASSERT_EQ(one, t->single); + std::string all(t->original); + ASSERT_EQ(RE2::GlobalReplace(&all, t->regexp, t->rewrite), t->greplace_count) + << "Got: " << all; + ASSERT_EQ(all, t->global); + } +} + +static void TestCheckRewriteString(const char* regexp, const char* rewrite, + bool expect_ok) { + std::string error; + RE2 exp(regexp); + bool actual_ok = exp.CheckRewriteString(rewrite, &error); + EXPECT_EQ(expect_ok, actual_ok) << " for " << rewrite << " error: " << error; +} + +TEST(CheckRewriteString, all) { + TestCheckRewriteString("abc", "foo", true); + TestCheckRewriteString("abc", "foo\\", false); + TestCheckRewriteString("abc", "foo\\0bar", true); + + TestCheckRewriteString("a(b)c", "foo", true); + TestCheckRewriteString("a(b)c", "foo\\0bar", true); + TestCheckRewriteString("a(b)c", "foo\\1bar", true); + TestCheckRewriteString("a(b)c", "foo\\2bar", false); + TestCheckRewriteString("a(b)c", "f\\\\2o\\1o", true); + + TestCheckRewriteString("a(b)(c)", "foo\\12", true); + TestCheckRewriteString("a(b)(c)", "f\\2o\\1o", true); + TestCheckRewriteString("a(b)(c)", "f\\oo\\1", false); +} + +TEST(RE2, Extract) { + std::string s; + + ASSERT_TRUE(RE2::Extract("boris@kremvax.ru", "(.*)@([^.]*)", "\\2!\\1", &s)); + ASSERT_EQ(s, "kremvax!boris"); + + ASSERT_TRUE(RE2::Extract("foo", ".*", "'\\0'", &s)); + ASSERT_EQ(s, "'foo'"); + // check that false match doesn't overwrite + ASSERT_FALSE(RE2::Extract("baz", "bar", "'\\0'", &s)); + ASSERT_EQ(s, "'foo'"); +} + +TEST(RE2, Consume) { + RE2 r("\\s*(\\w+)"); // matches a word, possibly proceeded by whitespace + std::string word; + + std::string s(" aaa b!@#$@#$cccc"); + StringPiece input(s); + + ASSERT_TRUE(RE2::Consume(&input, r, &word)); + ASSERT_EQ(word, "aaa") << " input: " << input; + ASSERT_TRUE(RE2::Consume(&input, r, &word)); + ASSERT_EQ(word, "b") << " input: " << input; + ASSERT_FALSE(RE2::Consume(&input, r, &word)) << " input: " << input; +} + +TEST(RE2, ConsumeN) { + const std::string s(" one two three 4"); + StringPiece input(s); + + RE2::Arg argv[2]; + const RE2::Arg* const args[2] = { &argv[0], &argv[1] }; + + // 0 arg + EXPECT_TRUE(RE2::ConsumeN(&input, "\\s*(\\w+)", args, 0)); // Skips "one". + + // 1 arg + std::string word; + argv[0] = &word; + EXPECT_TRUE(RE2::ConsumeN(&input, "\\s*(\\w+)", args, 1)); + EXPECT_EQ("two", word); + + // Multi-args + int n; + argv[1] = &n; + EXPECT_TRUE(RE2::ConsumeN(&input, "\\s*(\\w+)\\s*(\\d+)", args, 2)); + EXPECT_EQ("three", word); + EXPECT_EQ(4, n); +} + +TEST(RE2, FindAndConsume) { + RE2 r("(\\w+)"); // matches a word + std::string word; + + std::string s(" aaa b!@#$@#$cccc"); + StringPiece input(s); + + ASSERT_TRUE(RE2::FindAndConsume(&input, r, &word)); + ASSERT_EQ(word, "aaa"); + ASSERT_TRUE(RE2::FindAndConsume(&input, r, &word)); + ASSERT_EQ(word, "b"); + ASSERT_TRUE(RE2::FindAndConsume(&input, r, &word)); + ASSERT_EQ(word, "cccc"); + ASSERT_FALSE(RE2::FindAndConsume(&input, r, &word)); + + // Check that FindAndConsume works without any submatches. + // Earlier version used uninitialized data for + // length to consume. + input = "aaa"; + ASSERT_TRUE(RE2::FindAndConsume(&input, "aaa")); + ASSERT_EQ(input, ""); +} + +TEST(RE2, FindAndConsumeN) { + const std::string s(" one two three 4"); + StringPiece input(s); + + RE2::Arg argv[2]; + const RE2::Arg* const args[2] = { &argv[0], &argv[1] }; + + // 0 arg + EXPECT_TRUE(RE2::FindAndConsumeN(&input, "(\\w+)", args, 0)); // Skips "one". + + // 1 arg + std::string word; + argv[0] = &word; + EXPECT_TRUE(RE2::FindAndConsumeN(&input, "(\\w+)", args, 1)); + EXPECT_EQ("two", word); + + // Multi-args + int n; + argv[1] = &n; + EXPECT_TRUE(RE2::FindAndConsumeN(&input, "(\\w+)\\s*(\\d+)", args, 2)); + EXPECT_EQ("three", word); + EXPECT_EQ(4, n); +} + +TEST(RE2, MatchNumberPeculiarity) { + RE2 r("(foo)|(bar)|(baz)"); + std::string word1; + std::string word2; + std::string word3; + + ASSERT_TRUE(RE2::PartialMatch("foo", r, &word1, &word2, &word3)); + ASSERT_EQ(word1, "foo"); + ASSERT_EQ(word2, ""); + ASSERT_EQ(word3, ""); + ASSERT_TRUE(RE2::PartialMatch("bar", r, &word1, &word2, &word3)); + ASSERT_EQ(word1, ""); + ASSERT_EQ(word2, "bar"); + ASSERT_EQ(word3, ""); + ASSERT_TRUE(RE2::PartialMatch("baz", r, &word1, &word2, &word3)); + ASSERT_EQ(word1, ""); + ASSERT_EQ(word2, ""); + ASSERT_EQ(word3, "baz"); + ASSERT_FALSE(RE2::PartialMatch("f", r, &word1, &word2, &word3)); + + std::string a; + ASSERT_TRUE(RE2::FullMatch("hello", "(foo)|hello", &a)); + ASSERT_EQ(a, ""); +} + +TEST(RE2, Match) { + RE2 re("((\\w+):([0-9]+))"); // extracts host and port + StringPiece group[4]; + + // No match. + StringPiece s = "zyzzyva"; + ASSERT_FALSE( + re.Match(s, 0, s.size(), RE2::UNANCHORED, group, arraysize(group))); + + // Matches and extracts. + s = "a chrisr:9000 here"; + ASSERT_TRUE( + re.Match(s, 0, s.size(), RE2::UNANCHORED, group, arraysize(group))); + ASSERT_EQ(group[0], "chrisr:9000"); + ASSERT_EQ(group[1], "chrisr:9000"); + ASSERT_EQ(group[2], "chrisr"); + ASSERT_EQ(group[3], "9000"); + + std::string all, host; + int port; + ASSERT_TRUE(RE2::PartialMatch("a chrisr:9000 here", re, &all, &host, &port)); + ASSERT_EQ(all, "chrisr:9000"); + ASSERT_EQ(host, "chrisr"); + ASSERT_EQ(port, 9000); +} + +static void TestRecursion(int size, const char* pattern) { + // Fill up a string repeating the pattern given + std::string domain; + domain.resize(size); + size_t patlen = strlen(pattern); + for (int i = 0; i < size; i++) { + domain[i] = pattern[i % patlen]; + } + // Just make sure it doesn't crash due to too much recursion. + RE2 re("([a-zA-Z0-9]|-)+(\\.([a-zA-Z0-9]|-)+)*(\\.)?", RE2::Quiet); + RE2::FullMatch(domain, re); +} + +// A meta-quoted string, interpreted as a pattern, should always match +// the original unquoted string. +static void TestQuoteMeta(const std::string& unquoted, + const RE2::Options& options = RE2::DefaultOptions) { + std::string quoted = RE2::QuoteMeta(unquoted); + RE2 re(quoted, options); + EXPECT_TRUE(RE2::FullMatch(unquoted, re)) + << "Unquoted='" << unquoted << "', quoted='" << quoted << "'."; +} + +// A meta-quoted string, interpreted as a pattern, should always match +// the original unquoted string. +static void NegativeTestQuoteMeta( + const std::string& unquoted, const std::string& should_not_match, + const RE2::Options& options = RE2::DefaultOptions) { + std::string quoted = RE2::QuoteMeta(unquoted); + RE2 re(quoted, options); + EXPECT_FALSE(RE2::FullMatch(should_not_match, re)) + << "Unquoted='" << unquoted << "', quoted='" << quoted << "'."; +} + +// Tests that quoted meta characters match their original strings, +// and that a few things that shouldn't match indeed do not. +TEST(QuoteMeta, Simple) { + TestQuoteMeta("foo"); + TestQuoteMeta("foo.bar"); + TestQuoteMeta("foo\\.bar"); + TestQuoteMeta("[1-9]"); + TestQuoteMeta("1.5-2.0?"); + TestQuoteMeta("\\d"); + TestQuoteMeta("Who doesn't like ice cream?"); + TestQuoteMeta("((a|b)c?d*e+[f-h]i)"); + TestQuoteMeta("((?!)xxx).*yyy"); + TestQuoteMeta("(["); +} +TEST(QuoteMeta, SimpleNegative) { + NegativeTestQuoteMeta("foo", "bar"); + NegativeTestQuoteMeta("...", "bar"); + NegativeTestQuoteMeta("\\.", "."); + NegativeTestQuoteMeta("\\.", ".."); + NegativeTestQuoteMeta("(a)", "a"); + NegativeTestQuoteMeta("(a|b)", "a"); + NegativeTestQuoteMeta("(a|b)", "(a)"); + NegativeTestQuoteMeta("(a|b)", "a|b"); + NegativeTestQuoteMeta("[0-9]", "0"); + NegativeTestQuoteMeta("[0-9]", "0-9"); + NegativeTestQuoteMeta("[0-9]", "[9]"); + NegativeTestQuoteMeta("((?!)xxx)", "xxx"); +} + +TEST(QuoteMeta, Latin1) { + TestQuoteMeta("3\xb2 = 9", RE2::Latin1); +} + +TEST(QuoteMeta, UTF8) { + TestQuoteMeta("Plácido Domingo"); + TestQuoteMeta("xyz"); // No fancy utf8. + TestQuoteMeta("\xc2\xb0"); // 2-byte utf8 -- a degree symbol. + TestQuoteMeta("27\xc2\xb0 degrees"); // As a middle character. + TestQuoteMeta("\xe2\x80\xb3"); // 3-byte utf8 -- a double prime. + TestQuoteMeta("\xf0\x9d\x85\x9f"); // 4-byte utf8 -- a music note. + TestQuoteMeta("27\xc2\xb0"); // Interpreted as Latin-1, this should + // still work. + NegativeTestQuoteMeta("27\xc2\xb0", + "27\\\xc2\\\xb0"); // 2-byte utf8 -- a degree symbol. +} + +TEST(QuoteMeta, HasNull) { + std::string has_null; + + // string with one null character + has_null += '\0'; + TestQuoteMeta(has_null); + NegativeTestQuoteMeta(has_null, ""); + + // Don't want null-followed-by-'1' to be interpreted as '\01'. + has_null += '1'; + TestQuoteMeta(has_null); + NegativeTestQuoteMeta(has_null, "\1"); +} + +TEST(ProgramSize, BigProgram) { + RE2 re_simple("simple regexp"); + RE2 re_medium("medium.*regexp"); + RE2 re_complex("complex.{1,128}regexp"); + + ASSERT_GT(re_simple.ProgramSize(), 0); + ASSERT_GT(re_medium.ProgramSize(), re_simple.ProgramSize()); + ASSERT_GT(re_complex.ProgramSize(), re_medium.ProgramSize()); + + ASSERT_GT(re_simple.ReverseProgramSize(), 0); + ASSERT_GT(re_medium.ReverseProgramSize(), re_simple.ReverseProgramSize()); + ASSERT_GT(re_complex.ReverseProgramSize(), re_medium.ReverseProgramSize()); +} + +TEST(ProgramFanout, BigProgram) { + RE2 re1("(?:(?:(?:(?:(?:.)?){1})*)+)"); + RE2 re10("(?:(?:(?:(?:(?:.)?){10})*)+)"); + RE2 re100("(?:(?:(?:(?:(?:.)?){100})*)+)"); + RE2 re1000("(?:(?:(?:(?:(?:.)?){1000})*)+)"); + + std::map histogram; + + // 3 is the largest non-empty bucket and has 1 element. + ASSERT_EQ(3, re1.ProgramFanout(&histogram)); + ASSERT_EQ(1, histogram[3]); + + // 7 is the largest non-empty bucket and has 10 elements. + ASSERT_EQ(7, re10.ProgramFanout(&histogram)); + ASSERT_EQ(10, histogram[7]); + + // 10 is the largest non-empty bucket and has 100 elements. + ASSERT_EQ(10, re100.ProgramFanout(&histogram)); + ASSERT_EQ(100, histogram[10]); + + // 13 is the largest non-empty bucket and has 1000 elements. + ASSERT_EQ(13, re1000.ProgramFanout(&histogram)); + ASSERT_EQ(1000, histogram[13]); + + // 2 is the largest non-empty bucket and has 3 elements. + // This differs from the others due to how reverse `.' works. + ASSERT_EQ(2, re1.ReverseProgramFanout(&histogram)); + ASSERT_EQ(3, histogram[2]); + + // 5 is the largest non-empty bucket and has 10 elements. + ASSERT_EQ(5, re10.ReverseProgramFanout(&histogram)); + ASSERT_EQ(10, histogram[5]); + + // 9 is the largest non-empty bucket and has 100 elements. + ASSERT_EQ(9, re100.ReverseProgramFanout(&histogram)); + ASSERT_EQ(100, histogram[9]); + + // 12 is the largest non-empty bucket and has 1000 elements. + ASSERT_EQ(12, re1000.ReverseProgramFanout(&histogram)); + ASSERT_EQ(1000, histogram[12]); +} + +// Issue 956519: handling empty character sets was +// causing NULL dereference. This tests a few empty character sets. +// (The way to get an empty character set is to negate a full one.) +TEST(EmptyCharset, Fuzz) { + static const char *empties[] = { + "[^\\S\\s]", + "[^\\S[:space:]]", + "[^\\D\\d]", + "[^\\D[:digit:]]" + }; + for (size_t i = 0; i < arraysize(empties); i++) + ASSERT_FALSE(RE2(empties[i]).Match("abc", 0, 3, RE2::UNANCHORED, NULL, 0)); +} + +// Bitstate assumes that kInstFail instructions in +// alternations or capture groups have been "compiled away". +TEST(EmptyCharset, BitstateAssumptions) { + // Captures trigger use of Bitstate. + static const char *nop_empties[] = { + "((((()))))" "[^\\S\\s]?", + "((((()))))" "([^\\S\\s])?", + "((((()))))" "([^\\S\\s]|[^\\S\\s])?", + "((((()))))" "(([^\\S\\s]|[^\\S\\s])|)" + }; + StringPiece group[6]; + for (size_t i = 0; i < arraysize(nop_empties); i++) + ASSERT_TRUE(RE2(nop_empties[i]).Match("", 0, 0, RE2::UNANCHORED, group, 6)); +} + +// Test that named groups work correctly. +TEST(Capture, NamedGroups) { + { + RE2 re("(hello world)"); + ASSERT_EQ(re.NumberOfCapturingGroups(), 1); + const std::map& m = re.NamedCapturingGroups(); + ASSERT_EQ(m.size(), 0); + } + + { + RE2 re("(?Pexpr(?Pexpr)(?Pexpr))((expr)(?Pexpr))"); + ASSERT_EQ(re.NumberOfCapturingGroups(), 6); + const std::map& m = re.NamedCapturingGroups(); + ASSERT_EQ(m.size(), 4); + ASSERT_EQ(m.find("A")->second, 1); + ASSERT_EQ(m.find("B")->second, 2); + ASSERT_EQ(m.find("C")->second, 3); + ASSERT_EQ(m.find("D")->second, 6); // $4 and $5 are anonymous + } +} + +TEST(RE2, CapturedGroupTest) { + RE2 re("directions from (?P.*) to (?P.*)"); + int num_groups = re.NumberOfCapturingGroups(); + EXPECT_EQ(2, num_groups); + std::string args[4]; + RE2::Arg arg0(&args[0]); + RE2::Arg arg1(&args[1]); + RE2::Arg arg2(&args[2]); + RE2::Arg arg3(&args[3]); + + const RE2::Arg* const matches[4] = {&arg0, &arg1, &arg2, &arg3}; + EXPECT_TRUE(RE2::FullMatchN("directions from mountain view to san jose", + re, matches, num_groups)); + const std::map& named_groups = re.NamedCapturingGroups(); + EXPECT_TRUE(named_groups.find("S") != named_groups.end()); + EXPECT_TRUE(named_groups.find("D") != named_groups.end()); + + // The named group index is 1-based. + int source_group_index = named_groups.find("S")->second; + int destination_group_index = named_groups.find("D")->second; + EXPECT_EQ(1, source_group_index); + EXPECT_EQ(2, destination_group_index); + + // The args is zero-based. + EXPECT_EQ("mountain view", args[source_group_index - 1]); + EXPECT_EQ("san jose", args[destination_group_index - 1]); +} + +TEST(RE2, FullMatchWithNoArgs) { + ASSERT_TRUE(RE2::FullMatch("h", "h")); + ASSERT_TRUE(RE2::FullMatch("hello", "hello")); + ASSERT_TRUE(RE2::FullMatch("hello", "h.*o")); + ASSERT_FALSE(RE2::FullMatch("othello", "h.*o")); // Must be anchored at front + ASSERT_FALSE(RE2::FullMatch("hello!", "h.*o")); // Must be anchored at end +} + +TEST(RE2, PartialMatch) { + ASSERT_TRUE(RE2::PartialMatch("x", "x")); + ASSERT_TRUE(RE2::PartialMatch("hello", "h.*o")); + ASSERT_TRUE(RE2::PartialMatch("othello", "h.*o")); + ASSERT_TRUE(RE2::PartialMatch("hello!", "h.*o")); + ASSERT_TRUE(RE2::PartialMatch("x", "((((((((((((((((((((x))))))))))))))))))))")); +} + +TEST(RE2, PartialMatchN) { + RE2::Arg argv[2]; + const RE2::Arg* const args[2] = { &argv[0], &argv[1] }; + + // 0 arg + EXPECT_TRUE(RE2::PartialMatchN("hello", "e.*o", args, 0)); + EXPECT_FALSE(RE2::PartialMatchN("othello", "a.*o", args, 0)); + + // 1 arg + int i; + argv[0] = &i; + EXPECT_TRUE(RE2::PartialMatchN("1001 nights", "(\\d+)", args, 1)); + EXPECT_EQ(1001, i); + EXPECT_FALSE(RE2::PartialMatchN("three", "(\\d+)", args, 1)); + + // Multi-arg + std::string s; + argv[1] = &s; + EXPECT_TRUE(RE2::PartialMatchN("answer: 42:life", "(\\d+):(\\w+)", args, 2)); + EXPECT_EQ(42, i); + EXPECT_EQ("life", s); + EXPECT_FALSE(RE2::PartialMatchN("hi1", "(\\w+)(1)", args, 2)); +} + +TEST(RE2, FullMatchZeroArg) { + // Zero-arg + ASSERT_TRUE(RE2::FullMatch("1001", "\\d+")); +} + +TEST(RE2, FullMatchOneArg) { + int i; + + // Single-arg + ASSERT_TRUE(RE2::FullMatch("1001", "(\\d+)", &i)); + ASSERT_EQ(i, 1001); + ASSERT_TRUE(RE2::FullMatch("-123", "(-?\\d+)", &i)); + ASSERT_EQ(i, -123); + ASSERT_FALSE(RE2::FullMatch("10", "()\\d+", &i)); + ASSERT_FALSE( + RE2::FullMatch("1234567890123456789012345678901234567890", "(\\d+)", &i)); +} + +TEST(RE2, FullMatchIntegerArg) { + int i; + + // Digits surrounding integer-arg + ASSERT_TRUE(RE2::FullMatch("1234", "1(\\d*)4", &i)); + ASSERT_EQ(i, 23); + ASSERT_TRUE(RE2::FullMatch("1234", "(\\d)\\d+", &i)); + ASSERT_EQ(i, 1); + ASSERT_TRUE(RE2::FullMatch("-1234", "(-\\d)\\d+", &i)); + ASSERT_EQ(i, -1); + ASSERT_TRUE(RE2::PartialMatch("1234", "(\\d)", &i)); + ASSERT_EQ(i, 1); + ASSERT_TRUE(RE2::PartialMatch("-1234", "(-\\d)", &i)); + ASSERT_EQ(i, -1); +} + +TEST(RE2, FullMatchStringArg) { + std::string s; + // String-arg + ASSERT_TRUE(RE2::FullMatch("hello", "h(.*)o", &s)); + ASSERT_EQ(s, std::string("ell")); +} + +TEST(RE2, FullMatchStringPieceArg) { + int i; + // StringPiece-arg + StringPiece sp; + ASSERT_TRUE(RE2::FullMatch("ruby:1234", "(\\w+):(\\d+)", &sp, &i)); + ASSERT_EQ(sp.size(), 4); + ASSERT_TRUE(memcmp(sp.data(), "ruby", 4) == 0); + ASSERT_EQ(i, 1234); +} + +TEST(RE2, FullMatchMultiArg) { + int i; + std::string s; + // Multi-arg + ASSERT_TRUE(RE2::FullMatch("ruby:1234", "(\\w+):(\\d+)", &s, &i)); + ASSERT_EQ(s, std::string("ruby")); + ASSERT_EQ(i, 1234); +} + +TEST(RE2, FullMatchN) { + RE2::Arg argv[2]; + const RE2::Arg* const args[2] = { &argv[0], &argv[1] }; + + // 0 arg + EXPECT_TRUE(RE2::FullMatchN("hello", "h.*o", args, 0)); + EXPECT_FALSE(RE2::FullMatchN("othello", "h.*o", args, 0)); + + // 1 arg + int i; + argv[0] = &i; + EXPECT_TRUE(RE2::FullMatchN("1001", "(\\d+)", args, 1)); + EXPECT_EQ(1001, i); + EXPECT_FALSE(RE2::FullMatchN("three", "(\\d+)", args, 1)); + + // Multi-arg + std::string s; + argv[1] = &s; + EXPECT_TRUE(RE2::FullMatchN("42:life", "(\\d+):(\\w+)", args, 2)); + EXPECT_EQ(42, i); + EXPECT_EQ("life", s); + EXPECT_FALSE(RE2::FullMatchN("hi1", "(\\w+)(1)", args, 2)); +} + +TEST(RE2, FullMatchIgnoredArg) { + int i; + std::string s; + + // Old-school NULL should be ignored. + ASSERT_TRUE( + RE2::FullMatch("ruby:1234", "(\\w+)(:)(\\d+)", &s, (void*)NULL, &i)); + ASSERT_EQ(s, std::string("ruby")); + ASSERT_EQ(i, 1234); + + // C++11 nullptr should also be ignored. + ASSERT_TRUE(RE2::FullMatch("rubz:1235", "(\\w+)(:)(\\d+)", &s, nullptr, &i)); + ASSERT_EQ(s, std::string("rubz")); + ASSERT_EQ(i, 1235); +} + +TEST(RE2, FullMatchTypedNullArg) { + std::string s; + + // Ignore non-void* NULL arg + ASSERT_TRUE(RE2::FullMatch("hello", "he(.*)lo", (char*)NULL)); + ASSERT_TRUE(RE2::FullMatch("hello", "h(.*)o", (std::string*)NULL)); + ASSERT_TRUE(RE2::FullMatch("hello", "h(.*)o", (StringPiece*)NULL)); + ASSERT_TRUE(RE2::FullMatch("1234", "(.*)", (int*)NULL)); + ASSERT_TRUE(RE2::FullMatch("1234567890123456", "(.*)", (long long*)NULL)); + ASSERT_TRUE(RE2::FullMatch("123.4567890123456", "(.*)", (double*)NULL)); + ASSERT_TRUE(RE2::FullMatch("123.4567890123456", "(.*)", (float*)NULL)); + + // Fail on non-void* NULL arg if the match doesn't parse for the given type. + ASSERT_FALSE(RE2::FullMatch("hello", "h(.*)lo", &s, (char*)NULL)); + ASSERT_FALSE(RE2::FullMatch("hello", "(.*)", (int*)NULL)); + ASSERT_FALSE(RE2::FullMatch("1234567890123456", "(.*)", (int*)NULL)); + ASSERT_FALSE(RE2::FullMatch("hello", "(.*)", (double*)NULL)); + ASSERT_FALSE(RE2::FullMatch("hello", "(.*)", (float*)NULL)); +} + +// Check that numeric parsing code does not read past the end of +// the number being parsed. +// This implementation requires mmap(2) et al. and thus cannot +// be used unless they are available. +TEST(RE2, NULTerminated) { +#if defined(_POSIX_MAPPED_FILES) && _POSIX_MAPPED_FILES > 0 + char *v; + int x; + long pagesize = sysconf(_SC_PAGE_SIZE); + +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif + v = static_cast(mmap(NULL, 2*pagesize, PROT_READ|PROT_WRITE, + MAP_ANONYMOUS|MAP_PRIVATE, -1, 0)); + ASSERT_TRUE(v != reinterpret_cast(-1)); + LOG(INFO) << "Memory at " << (void*)v; + ASSERT_EQ(munmap(v + pagesize, pagesize), 0) << " error " << errno; + v[pagesize - 1] = '1'; + + x = 0; + ASSERT_TRUE(RE2::FullMatch(StringPiece(v + pagesize - 1, 1), "(.*)", &x)); + ASSERT_EQ(x, 1); +#endif +} + +TEST(RE2, FullMatchTypeTests) { + // Type tests + std::string zeros(1000, '0'); + { + char c; + ASSERT_TRUE(RE2::FullMatch("Hello", "(H)ello", &c)); + ASSERT_EQ(c, 'H'); + } + { + unsigned char c; + ASSERT_TRUE(RE2::FullMatch("Hello", "(H)ello", &c)); + ASSERT_EQ(c, static_cast('H')); + } + { + int16_t v; + ASSERT_TRUE(RE2::FullMatch("100", "(-?\\d+)", &v)); ASSERT_EQ(v, 100); + ASSERT_TRUE(RE2::FullMatch("-100", "(-?\\d+)", &v)); ASSERT_EQ(v, -100); + ASSERT_TRUE(RE2::FullMatch("32767", "(-?\\d+)", &v)); ASSERT_EQ(v, 32767); + ASSERT_TRUE(RE2::FullMatch("-32768", "(-?\\d+)", &v)); ASSERT_EQ(v, -32768); + ASSERT_FALSE(RE2::FullMatch("-32769", "(-?\\d+)", &v)); + ASSERT_FALSE(RE2::FullMatch("32768", "(-?\\d+)", &v)); + } + { + uint16_t v; + ASSERT_TRUE(RE2::FullMatch("100", "(\\d+)", &v)); ASSERT_EQ(v, 100); + ASSERT_TRUE(RE2::FullMatch("32767", "(\\d+)", &v)); ASSERT_EQ(v, 32767); + ASSERT_TRUE(RE2::FullMatch("65535", "(\\d+)", &v)); ASSERT_EQ(v, 65535); + ASSERT_FALSE(RE2::FullMatch("65536", "(\\d+)", &v)); + } + { + int32_t v; + static const int32_t max = INT32_C(0x7fffffff); + static const int32_t min = -max - 1; + ASSERT_TRUE(RE2::FullMatch("100", "(-?\\d+)", &v)); ASSERT_EQ(v, 100); + ASSERT_TRUE(RE2::FullMatch("-100", "(-?\\d+)", &v)); ASSERT_EQ(v, -100); + ASSERT_TRUE(RE2::FullMatch("2147483647", "(-?\\d+)", &v)); ASSERT_EQ(v, max); + ASSERT_TRUE(RE2::FullMatch("-2147483648", "(-?\\d+)", &v)); ASSERT_EQ(v, min); + ASSERT_FALSE(RE2::FullMatch("-2147483649", "(-?\\d+)", &v)); + ASSERT_FALSE(RE2::FullMatch("2147483648", "(-?\\d+)", &v)); + + ASSERT_TRUE(RE2::FullMatch(zeros + "2147483647", "(-?\\d+)", &v)); + ASSERT_EQ(v, max); + ASSERT_TRUE(RE2::FullMatch("-" + zeros + "2147483648", "(-?\\d+)", &v)); + ASSERT_EQ(v, min); + + ASSERT_FALSE(RE2::FullMatch("-" + zeros + "2147483649", "(-?\\d+)", &v)); + ASSERT_TRUE(RE2::FullMatch("0x7fffffff", "(.*)", RE2::CRadix(&v))); + ASSERT_EQ(v, max); + ASSERT_FALSE(RE2::FullMatch("000x7fffffff", "(.*)", RE2::CRadix(&v))); + } + { + uint32_t v; + static const uint32_t max = UINT32_C(0xffffffff); + ASSERT_TRUE(RE2::FullMatch("100", "(\\d+)", &v)); ASSERT_EQ(v, 100); + ASSERT_TRUE(RE2::FullMatch("4294967295", "(\\d+)", &v)); ASSERT_EQ(v, max); + ASSERT_FALSE(RE2::FullMatch("4294967296", "(\\d+)", &v)); + ASSERT_FALSE(RE2::FullMatch("-1", "(\\d+)", &v)); + + ASSERT_TRUE(RE2::FullMatch(zeros + "4294967295", "(\\d+)", &v)); ASSERT_EQ(v, max); + } + { + int64_t v; + static const int64_t max = INT64_C(0x7fffffffffffffff); + static const int64_t min = -max - 1; + std::string str; + + ASSERT_TRUE(RE2::FullMatch("100", "(-?\\d+)", &v)); ASSERT_EQ(v, 100); + ASSERT_TRUE(RE2::FullMatch("-100", "(-?\\d+)", &v)); ASSERT_EQ(v, -100); + + str = std::to_string(max); + ASSERT_TRUE(RE2::FullMatch(str, "(-?\\d+)", &v)); ASSERT_EQ(v, max); + + str = std::to_string(min); + ASSERT_TRUE(RE2::FullMatch(str, "(-?\\d+)", &v)); ASSERT_EQ(v, min); + + str = std::to_string(max); + ASSERT_NE(str.back(), '9'); + str.back()++; + ASSERT_FALSE(RE2::FullMatch(str, "(-?\\d+)", &v)); + + str = std::to_string(min); + ASSERT_NE(str.back(), '9'); + str.back()++; + ASSERT_FALSE(RE2::FullMatch(str, "(-?\\d+)", &v)); + } + { + uint64_t v; + int64_t v2; + static const uint64_t max = UINT64_C(0xffffffffffffffff); + std::string str; + + ASSERT_TRUE(RE2::FullMatch("100", "(-?\\d+)", &v)); ASSERT_EQ(v, 100); + ASSERT_TRUE(RE2::FullMatch("-100", "(-?\\d+)", &v2)); ASSERT_EQ(v2, -100); + + str = std::to_string(max); + ASSERT_TRUE(RE2::FullMatch(str, "(-?\\d+)", &v)); ASSERT_EQ(v, max); + + ASSERT_NE(str.back(), '9'); + str.back()++; + ASSERT_FALSE(RE2::FullMatch(str, "(-?\\d+)", &v)); + } +} + +TEST(RE2, FloatingPointFullMatchTypes) { + std::string zeros(1000, '0'); + { + float v; + ASSERT_TRUE(RE2::FullMatch("100", "(.*)", &v)); ASSERT_EQ(v, 100); + ASSERT_TRUE(RE2::FullMatch("-100.", "(.*)", &v)); ASSERT_EQ(v, -100); + ASSERT_TRUE(RE2::FullMatch("1e23", "(.*)", &v)); ASSERT_EQ(v, float(1e23)); + ASSERT_TRUE(RE2::FullMatch(" 100", "(.*)", &v)); ASSERT_EQ(v, 100); + + ASSERT_TRUE(RE2::FullMatch(zeros + "1e23", "(.*)", &v)); + ASSERT_EQ(v, float(1e23)); + + // 6700000000081920.1 is an edge case. + // 6700000000081920 is exactly halfway between + // two float32s, so the .1 should make it round up. + // However, the .1 is outside the precision possible with + // a float64: the nearest float64 is 6700000000081920. + // So if the code uses strtod and then converts to float32, + // round-to-even will make it round down instead of up. + // To pass the test, the parser must call strtof directly. + // This test case is carefully chosen to use only a 17-digit + // number, since C does not guarantee to get the correctly + // rounded answer for strtod and strtof unless the input is + // short. + // + // This is known to fail on Cygwin and MinGW due to a broken + // implementation of strtof(3). And apparently MSVC too. Sigh. +#if !defined(_MSC_VER) && !defined(__CYGWIN__) && !defined(__MINGW32__) + ASSERT_TRUE(RE2::FullMatch("0.1", "(.*)", &v)); + ASSERT_EQ(v, 0.1f) << StringPrintf("%.8g != %.8g", v, 0.1f); + ASSERT_TRUE(RE2::FullMatch("6700000000081920.1", "(.*)", &v)); + ASSERT_EQ(v, 6700000000081920.1f) + << StringPrintf("%.8g != %.8g", v, 6700000000081920.1f); +#endif + } + { + double v; + ASSERT_TRUE(RE2::FullMatch("100", "(.*)", &v)); ASSERT_EQ(v, 100); + ASSERT_TRUE(RE2::FullMatch("-100.", "(.*)", &v)); ASSERT_EQ(v, -100); + ASSERT_TRUE(RE2::FullMatch("1e23", "(.*)", &v)); ASSERT_EQ(v, 1e23); + ASSERT_TRUE(RE2::FullMatch(zeros + "1e23", "(.*)", &v)); + ASSERT_EQ(v, double(1e23)); + + ASSERT_TRUE(RE2::FullMatch("0.1", "(.*)", &v)); + ASSERT_EQ(v, 0.1) << StringPrintf("%.17g != %.17g", v, 0.1); + ASSERT_TRUE(RE2::FullMatch("1.00000005960464485", "(.*)", &v)); + ASSERT_EQ(v, 1.0000000596046448) + << StringPrintf("%.17g != %.17g", v, 1.0000000596046448); + } +} + +TEST(RE2, FullMatchAnchored) { + int i; + // Check that matching is fully anchored + ASSERT_FALSE(RE2::FullMatch("x1001", "(\\d+)", &i)); + ASSERT_FALSE(RE2::FullMatch("1001x", "(\\d+)", &i)); + ASSERT_TRUE(RE2::FullMatch("x1001", "x(\\d+)", &i)); ASSERT_EQ(i, 1001); + ASSERT_TRUE(RE2::FullMatch("1001x", "(\\d+)x", &i)); ASSERT_EQ(i, 1001); +} + +TEST(RE2, FullMatchBraces) { + // Braces + ASSERT_TRUE(RE2::FullMatch("0abcd", "[0-9a-f+.-]{5,}")); + ASSERT_TRUE(RE2::FullMatch("0abcde", "[0-9a-f+.-]{5,}")); + ASSERT_FALSE(RE2::FullMatch("0abc", "[0-9a-f+.-]{5,}")); +} + +TEST(RE2, Complicated) { + // Complicated RE2 + ASSERT_TRUE(RE2::FullMatch("foo", "foo|bar|[A-Z]")); + ASSERT_TRUE(RE2::FullMatch("bar", "foo|bar|[A-Z]")); + ASSERT_TRUE(RE2::FullMatch("X", "foo|bar|[A-Z]")); + ASSERT_FALSE(RE2::FullMatch("XY", "foo|bar|[A-Z]")); +} + +TEST(RE2, FullMatchEnd) { + // Check full-match handling (needs '$' tacked on internally) + ASSERT_TRUE(RE2::FullMatch("fo", "fo|foo")); + ASSERT_TRUE(RE2::FullMatch("foo", "fo|foo")); + ASSERT_TRUE(RE2::FullMatch("fo", "fo|foo$")); + ASSERT_TRUE(RE2::FullMatch("foo", "fo|foo$")); + ASSERT_TRUE(RE2::FullMatch("foo", "foo$")); + ASSERT_FALSE(RE2::FullMatch("foo$bar", "foo\\$")); + ASSERT_FALSE(RE2::FullMatch("fox", "fo|bar")); + + // Uncomment the following if we change the handling of '$' to + // prevent it from matching a trailing newline + if (false) { + // Check that we don't get bitten by pcre's special handling of a + // '\n' at the end of the string matching '$' + ASSERT_FALSE(RE2::PartialMatch("foo\n", "foo$")); + } +} + +TEST(RE2, FullMatchArgCount) { + // Number of args + int a[16]; + ASSERT_TRUE(RE2::FullMatch("", "")); + + memset(a, 0, sizeof(0)); + ASSERT_TRUE(RE2::FullMatch("1", "(\\d){1}", &a[0])); + ASSERT_EQ(a[0], 1); + + memset(a, 0, sizeof(0)); + ASSERT_TRUE(RE2::FullMatch("12", "(\\d)(\\d)", &a[0], &a[1])); + ASSERT_EQ(a[0], 1); + ASSERT_EQ(a[1], 2); + + memset(a, 0, sizeof(0)); + ASSERT_TRUE(RE2::FullMatch("123", "(\\d)(\\d)(\\d)", &a[0], &a[1], &a[2])); + ASSERT_EQ(a[0], 1); + ASSERT_EQ(a[1], 2); + ASSERT_EQ(a[2], 3); + + memset(a, 0, sizeof(0)); + ASSERT_TRUE(RE2::FullMatch("1234", "(\\d)(\\d)(\\d)(\\d)", &a[0], &a[1], + &a[2], &a[3])); + ASSERT_EQ(a[0], 1); + ASSERT_EQ(a[1], 2); + ASSERT_EQ(a[2], 3); + ASSERT_EQ(a[3], 4); + + memset(a, 0, sizeof(0)); + ASSERT_TRUE(RE2::FullMatch("12345", "(\\d)(\\d)(\\d)(\\d)(\\d)", &a[0], &a[1], + &a[2], &a[3], &a[4])); + ASSERT_EQ(a[0], 1); + ASSERT_EQ(a[1], 2); + ASSERT_EQ(a[2], 3); + ASSERT_EQ(a[3], 4); + ASSERT_EQ(a[4], 5); + + memset(a, 0, sizeof(0)); + ASSERT_TRUE(RE2::FullMatch("123456", "(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)", &a[0], + &a[1], &a[2], &a[3], &a[4], &a[5])); + ASSERT_EQ(a[0], 1); + ASSERT_EQ(a[1], 2); + ASSERT_EQ(a[2], 3); + ASSERT_EQ(a[3], 4); + ASSERT_EQ(a[4], 5); + ASSERT_EQ(a[5], 6); + + memset(a, 0, sizeof(0)); + ASSERT_TRUE(RE2::FullMatch("1234567", "(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)", + &a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6])); + ASSERT_EQ(a[0], 1); + ASSERT_EQ(a[1], 2); + ASSERT_EQ(a[2], 3); + ASSERT_EQ(a[3], 4); + ASSERT_EQ(a[4], 5); + ASSERT_EQ(a[5], 6); + ASSERT_EQ(a[6], 7); + + memset(a, 0, sizeof(0)); + ASSERT_TRUE(RE2::FullMatch("1234567890123456", + "(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)" + "(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)", + &a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6], + &a[7], &a[8], &a[9], &a[10], &a[11], &a[12], + &a[13], &a[14], &a[15])); + ASSERT_EQ(a[0], 1); + ASSERT_EQ(a[1], 2); + ASSERT_EQ(a[2], 3); + ASSERT_EQ(a[3], 4); + ASSERT_EQ(a[4], 5); + ASSERT_EQ(a[5], 6); + ASSERT_EQ(a[6], 7); + ASSERT_EQ(a[7], 8); + ASSERT_EQ(a[8], 9); + ASSERT_EQ(a[9], 0); + ASSERT_EQ(a[10], 1); + ASSERT_EQ(a[11], 2); + ASSERT_EQ(a[12], 3); + ASSERT_EQ(a[13], 4); + ASSERT_EQ(a[14], 5); + ASSERT_EQ(a[15], 6); +} + +TEST(RE2, Accessors) { + // Check the pattern() accessor + { + const std::string kPattern = "http://([^/]+)/.*"; + const RE2 re(kPattern); + ASSERT_EQ(kPattern, re.pattern()); + } + + // Check RE2 error field. + { + RE2 re("foo"); + ASSERT_TRUE(re.error().empty()); // Must have no error + ASSERT_TRUE(re.ok()); + ASSERT_EQ(re.error_code(), RE2::NoError); + } +} + +TEST(RE2, UTF8) { + // Check UTF-8 handling + // Three Japanese characters (nihongo) + const char utf8_string[] = { + (char)0xe6, (char)0x97, (char)0xa5, // 65e5 + (char)0xe6, (char)0x9c, (char)0xac, // 627c + (char)0xe8, (char)0xaa, (char)0x9e, // 8a9e + 0 + }; + const char utf8_pattern[] = { + '.', + (char)0xe6, (char)0x9c, (char)0xac, // 627c + '.', + 0 + }; + + // Both should match in either mode, bytes or UTF-8 + RE2 re_test1(".........", RE2::Latin1); + ASSERT_TRUE(RE2::FullMatch(utf8_string, re_test1)); + RE2 re_test2("..."); + ASSERT_TRUE(RE2::FullMatch(utf8_string, re_test2)); + + // Check that '.' matches one byte or UTF-8 character + // according to the mode. + std::string s; + RE2 re_test3("(.)", RE2::Latin1); + ASSERT_TRUE(RE2::PartialMatch(utf8_string, re_test3, &s)); + ASSERT_EQ(s, std::string("\xe6")); + RE2 re_test4("(.)"); + ASSERT_TRUE(RE2::PartialMatch(utf8_string, re_test4, &s)); + ASSERT_EQ(s, std::string("\xe6\x97\xa5")); + + // Check that string matches itself in either mode + RE2 re_test5(utf8_string, RE2::Latin1); + ASSERT_TRUE(RE2::FullMatch(utf8_string, re_test5)); + RE2 re_test6(utf8_string); + ASSERT_TRUE(RE2::FullMatch(utf8_string, re_test6)); + + // Check that pattern matches string only in UTF8 mode + RE2 re_test7(utf8_pattern, RE2::Latin1); + ASSERT_FALSE(RE2::FullMatch(utf8_string, re_test7)); + RE2 re_test8(utf8_pattern); + ASSERT_TRUE(RE2::FullMatch(utf8_string, re_test8)); +} + +TEST(RE2, UngreedyUTF8) { + // Check that ungreedy, UTF8 regular expressions don't match when they + // oughtn't -- see bug 82246. + { + // This code always worked. + const char* pattern = "\\w+X"; + const std::string target = "a aX"; + RE2 match_sentence(pattern, RE2::Latin1); + RE2 match_sentence_re(pattern); + + ASSERT_FALSE(RE2::FullMatch(target, match_sentence)); + ASSERT_FALSE(RE2::FullMatch(target, match_sentence_re)); + } + { + const char* pattern = "(?U)\\w+X"; + const std::string target = "a aX"; + RE2 match_sentence(pattern, RE2::Latin1); + ASSERT_EQ(match_sentence.error(), ""); + RE2 match_sentence_re(pattern); + + ASSERT_FALSE(RE2::FullMatch(target, match_sentence)); + ASSERT_FALSE(RE2::FullMatch(target, match_sentence_re)); + } +} + +TEST(RE2, Rejects) { + { + RE2 re("a\\1", RE2::Quiet); + ASSERT_FALSE(re.ok()); } + { + RE2 re("a[x", RE2::Quiet); + ASSERT_FALSE(re.ok()); + } + { + RE2 re("a[z-a]", RE2::Quiet); + ASSERT_FALSE(re.ok()); + } + { + RE2 re("a[[:foobar:]]", RE2::Quiet); + ASSERT_FALSE(re.ok()); + } + { + RE2 re("a(b", RE2::Quiet); + ASSERT_FALSE(re.ok()); + } + { + RE2 re("a\\", RE2::Quiet); + ASSERT_FALSE(re.ok()); + } +} + +TEST(RE2, NoCrash) { + // Test that using a bad regexp doesn't crash. + { + RE2 re("a\\", RE2::Quiet); + ASSERT_FALSE(re.ok()); + ASSERT_FALSE(RE2::PartialMatch("a\\b", re)); + } + + // Test that using an enormous regexp doesn't crash + { + RE2 re("(((.{100}){100}){100}){100}", RE2::Quiet); + ASSERT_FALSE(re.ok()); + ASSERT_FALSE(RE2::PartialMatch("aaa", re)); + } + + // Test that a crazy regexp still compiles and runs. + { + RE2 re(".{512}x", RE2::Quiet); + ASSERT_TRUE(re.ok()); + std::string s; + s.append(515, 'c'); + s.append("x"); + ASSERT_TRUE(RE2::PartialMatch(s, re)); + } +} + +TEST(RE2, Recursion) { + // Test that recursion is stopped. + // This test is PCRE-legacy -- there's no recursion in RE2. + int bytes = 15 * 1024; // enough to crash PCRE + TestRecursion(bytes, "."); + TestRecursion(bytes, "a"); + TestRecursion(bytes, "a."); + TestRecursion(bytes, "ab."); + TestRecursion(bytes, "abc."); +} + +TEST(RE2, BigCountedRepetition) { + // Test that counted repetition works, given tons of memory. + RE2::Options opt; + opt.set_max_mem(256<<20); + + RE2 re(".{512}x", opt); + ASSERT_TRUE(re.ok()); + std::string s; + s.append(515, 'c'); + s.append("x"); + ASSERT_TRUE(RE2::PartialMatch(s, re)); +} + +TEST(RE2, DeepRecursion) { + // Test for deep stack recursion. This would fail with a + // segmentation violation due to stack overflow before pcre was + // patched. + // Again, a PCRE legacy test. RE2 doesn't recurse. + std::string comment("x*"); + std::string a(131072, 'a'); + comment += a; + comment += "*x"; + RE2 re("((?:\\s|xx.*\n|x[*](?:\n|.)*?[*]x)*)"); + ASSERT_TRUE(RE2::FullMatch(comment, re)); +} + +// Suggested by Josh Hyman. Failed when SearchOnePass was +// not implementing case-folding. +TEST(CaseInsensitive, MatchAndConsume) { + std::string result; + std::string text = "A fish named *Wanda*"; + StringPiece sp(text); + + EXPECT_TRUE(RE2::PartialMatch(sp, "(?i)([wand]{5})", &result)); + EXPECT_TRUE(RE2::FindAndConsume(&sp, "(?i)([wand]{5})", &result)); +} + +// RE2 should permit implicit conversions from string, StringPiece, const char*, +// and C string literals. +TEST(RE2, ImplicitConversions) { + std::string re_string("."); + StringPiece re_stringpiece("."); + const char* re_cstring = "."; + EXPECT_TRUE(RE2::PartialMatch("e", re_string)); + EXPECT_TRUE(RE2::PartialMatch("e", re_stringpiece)); + EXPECT_TRUE(RE2::PartialMatch("e", re_cstring)); + EXPECT_TRUE(RE2::PartialMatch("e", ".")); +} + +// Bugs introduced by 8622304 +TEST(RE2, CL8622304) { + // reported by ingow + std::string dir; + EXPECT_TRUE(RE2::FullMatch("D", "([^\\\\])")); // ok + EXPECT_TRUE(RE2::FullMatch("D", "([^\\\\])", &dir)); // fails + + // reported by jacobsa + std::string key, val; + EXPECT_TRUE(RE2::PartialMatch("bar:1,0x2F,030,4,5;baz:true;fooby:false,true", + "(\\w+)(?::((?:[^;\\\\]|\\\\.)*))?;?", + &key, + &val)); + EXPECT_EQ(key, "bar"); + EXPECT_EQ(val, "1,0x2F,030,4,5"); +} + + +// Check that RE2 returns correct regexp pieces on error. +// In particular, make sure it returns whole runes +// and that it always reports invalid UTF-8. +// Also check that Perl error flag piece is big enough. +static struct ErrorTest { + const char *regexp; + const char *error; +} error_tests[] = { + { "ab\\αcd", "\\α" }, + { "ef\\x☺01", "\\x☺0" }, + { "gh\\x1☺01", "\\x1☺" }, + { "ij\\x1", "\\x1" }, + { "kl\\x", "\\x" }, + { "uv\\x{0000☺}", "\\x{0000☺" }, + { "wx\\p{ABC", "\\p{ABC" }, + { "yz(?smiUX:abc)", "(?smiUX" }, // used to return (?s but the error is X + { "aa(?sm☺i", "(?sm☺" }, + { "bb[abc", "[abc" }, + + { "mn\\x1\377", "" }, // no argument string returned for invalid UTF-8 + { "op\377qr", "" }, + { "st\\x{00000\377", "" }, + { "zz\\p{\377}", "" }, + { "zz\\x{00\377}", "" }, + { "zz(?Pabc)", "" }, +}; +TEST(RE2, ErrorArgs) { + for (size_t i = 0; i < arraysize(error_tests); i++) { + RE2 re(error_tests[i].regexp, RE2::Quiet); + EXPECT_FALSE(re.ok()); + EXPECT_EQ(re.error_arg(), error_tests[i].error) << re.error(); + } +} + +// Check that "never match \n" mode never matches \n. +static struct NeverTest { + const char* regexp; + const char* text; + const char* match; +} never_tests[] = { + { "(.*)", "abc\ndef\nghi\n", "abc" }, + { "(?s)(abc.*def)", "abc\ndef\n", NULL }, + { "(abc(.|\n)*def)", "abc\ndef\n", NULL }, + { "(abc[^x]*def)", "abc\ndef\n", NULL }, + { "(abc[^x]*def)", "abczzzdef\ndef\n", "abczzzdef" }, +}; +TEST(RE2, NeverNewline) { + RE2::Options opt; + opt.set_never_nl(true); + for (size_t i = 0; i < arraysize(never_tests); i++) { + const NeverTest& t = never_tests[i]; + RE2 re(t.regexp, opt); + if (t.match == NULL) { + EXPECT_FALSE(re.PartialMatch(t.text, re)); + } else { + StringPiece m; + EXPECT_TRUE(re.PartialMatch(t.text, re, &m)); + EXPECT_EQ(m, t.match); + } + } +} + +// Check that dot_nl option works. +TEST(RE2, DotNL) { + RE2::Options opt; + opt.set_dot_nl(true); + EXPECT_TRUE(RE2::PartialMatch("\n", RE2(".", opt))); + EXPECT_FALSE(RE2::PartialMatch("\n", RE2("(?-s).", opt))); + opt.set_never_nl(true); + EXPECT_FALSE(RE2::PartialMatch("\n", RE2(".", opt))); +} + +// Check that there are no capturing groups in "never capture" mode. +TEST(RE2, NeverCapture) { + RE2::Options opt; + opt.set_never_capture(true); + RE2 re("(r)(e)", opt); + EXPECT_EQ(0, re.NumberOfCapturingGroups()); +} + +// Bitstate bug was looking at submatch[0] even if nsubmatch == 0. +// Triggered by a failed DFA search falling back to Bitstate when +// using Match with a NULL submatch set. Bitstate tried to read +// the submatch[0] entry even if nsubmatch was 0. +TEST(RE2, BitstateCaptureBug) { + RE2::Options opt; + opt.set_max_mem(20000); + RE2 re("(_________$)", opt); + StringPiece s = "xxxxxxxxxxxxxxxxxxxxxxxxxx_________x"; + EXPECT_FALSE(re.Match(s, 0, s.size(), RE2::UNANCHORED, NULL, 0)); +} + +// C++ version of bug 609710. +TEST(RE2, UnicodeClasses) { + const std::string str = "ABCDEFGHI譚永鋒"; + std::string a, b, c; + + EXPECT_TRUE(RE2::FullMatch("A", "\\p{L}")); + EXPECT_TRUE(RE2::FullMatch("A", "\\p{Lu}")); + EXPECT_FALSE(RE2::FullMatch("A", "\\p{Ll}")); + EXPECT_FALSE(RE2::FullMatch("A", "\\P{L}")); + EXPECT_FALSE(RE2::FullMatch("A", "\\P{Lu}")); + EXPECT_TRUE(RE2::FullMatch("A", "\\P{Ll}")); + + EXPECT_TRUE(RE2::FullMatch("譚", "\\p{L}")); + EXPECT_FALSE(RE2::FullMatch("譚", "\\p{Lu}")); + EXPECT_FALSE(RE2::FullMatch("譚", "\\p{Ll}")); + EXPECT_FALSE(RE2::FullMatch("譚", "\\P{L}")); + EXPECT_TRUE(RE2::FullMatch("譚", "\\P{Lu}")); + EXPECT_TRUE(RE2::FullMatch("譚", "\\P{Ll}")); + + EXPECT_TRUE(RE2::FullMatch("永", "\\p{L}")); + EXPECT_FALSE(RE2::FullMatch("永", "\\p{Lu}")); + EXPECT_FALSE(RE2::FullMatch("永", "\\p{Ll}")); + EXPECT_FALSE(RE2::FullMatch("永", "\\P{L}")); + EXPECT_TRUE(RE2::FullMatch("永", "\\P{Lu}")); + EXPECT_TRUE(RE2::FullMatch("永", "\\P{Ll}")); + + EXPECT_TRUE(RE2::FullMatch("鋒", "\\p{L}")); + EXPECT_FALSE(RE2::FullMatch("鋒", "\\p{Lu}")); + EXPECT_FALSE(RE2::FullMatch("鋒", "\\p{Ll}")); + EXPECT_FALSE(RE2::FullMatch("鋒", "\\P{L}")); + EXPECT_TRUE(RE2::FullMatch("鋒", "\\P{Lu}")); + EXPECT_TRUE(RE2::FullMatch("鋒", "\\P{Ll}")); + + EXPECT_TRUE(RE2::PartialMatch(str, "(.).*?(.).*?(.)", &a, &b, &c)); + EXPECT_EQ("A", a); + EXPECT_EQ("B", b); + EXPECT_EQ("C", c); + + EXPECT_TRUE(RE2::PartialMatch(str, "(.).*?([\\p{L}]).*?(.)", &a, &b, &c)); + EXPECT_EQ("A", a); + EXPECT_EQ("B", b); + EXPECT_EQ("C", c); + + EXPECT_FALSE(RE2::PartialMatch(str, "\\P{L}")); + + EXPECT_TRUE(RE2::PartialMatch(str, "(.).*?([\\p{Lu}]).*?(.)", &a, &b, &c)); + EXPECT_EQ("A", a); + EXPECT_EQ("B", b); + EXPECT_EQ("C", c); + + EXPECT_FALSE(RE2::PartialMatch(str, "[^\\p{Lu}\\p{Lo}]")); + + EXPECT_TRUE(RE2::PartialMatch(str, ".*(.).*?([\\p{Lu}\\p{Lo}]).*?(.)", &a, &b, &c)); + EXPECT_EQ("譚", a); + EXPECT_EQ("永", b); + EXPECT_EQ("鋒", c); +} + +TEST(RE2, LazyRE2) { + // Test with and without options. + static LazyRE2 a = {"a"}; + static LazyRE2 b = {"b", RE2::Latin1}; + + EXPECT_EQ("a", a->pattern()); + EXPECT_EQ(RE2::Options::EncodingUTF8, a->options().encoding()); + + EXPECT_EQ("b", b->pattern()); + EXPECT_EQ(RE2::Options::EncodingLatin1, b->options().encoding()); +} + +// Bug reported by saito. 2009/02/17 +TEST(RE2, NullVsEmptyString) { + RE2 re(".*"); + EXPECT_TRUE(re.ok()); + + StringPiece null; + EXPECT_TRUE(RE2::FullMatch(null, re)); + + StringPiece empty(""); + EXPECT_TRUE(RE2::FullMatch(empty, re)); +} + +// Similar to the previous test, check that the null string and the empty +// string both match, but also that the null string can only provide null +// submatches whereas the empty string can also provide empty submatches. +TEST(RE2, NullVsEmptyStringSubmatches) { + RE2 re("()|(foo)"); + EXPECT_TRUE(re.ok()); + + // matches[0] is overall match, [1] is (), [2] is (foo), [3] is nonexistent. + StringPiece matches[4]; + + for (size_t i = 0; i < arraysize(matches); i++) + matches[i] = "bar"; + + StringPiece null; + EXPECT_TRUE(re.Match(null, 0, null.size(), RE2::UNANCHORED, + matches, arraysize(matches))); + for (size_t i = 0; i < arraysize(matches); i++) { + EXPECT_TRUE(matches[i].data() == NULL); // always null + EXPECT_TRUE(matches[i].empty()); + } + + for (size_t i = 0; i < arraysize(matches); i++) + matches[i] = "bar"; + + StringPiece empty(""); + EXPECT_TRUE(re.Match(empty, 0, empty.size(), RE2::UNANCHORED, + matches, arraysize(matches))); + EXPECT_TRUE(matches[0].data() != NULL); // empty, not null + EXPECT_TRUE(matches[0].empty()); + EXPECT_TRUE(matches[1].data() != NULL); // empty, not null + EXPECT_TRUE(matches[1].empty()); + EXPECT_TRUE(matches[2].data() == NULL); + EXPECT_TRUE(matches[2].empty()); + EXPECT_TRUE(matches[3].data() == NULL); + EXPECT_TRUE(matches[3].empty()); +} + +// Issue 1816809 +TEST(RE2, Bug1816809) { + RE2 re("(((((llx((-3)|(4)))(;(llx((-3)|(4))))*))))"); + StringPiece piece("llx-3;llx4"); + std::string x; + EXPECT_TRUE(RE2::Consume(&piece, re, &x)); +} + +// Issue 3061120 +TEST(RE2, Bug3061120) { + RE2 re("(?i)\\W"); + EXPECT_FALSE(RE2::PartialMatch("x", re)); // always worked + EXPECT_FALSE(RE2::PartialMatch("k", re)); // broke because of kelvin + EXPECT_FALSE(RE2::PartialMatch("s", re)); // broke because of latin long s +} + +TEST(RE2, CapturingGroupNames) { + // Opening parentheses annotated with group IDs: + // 12 3 45 6 7 + RE2 re("((abc)(?P)|((e+)(?P.*)(?Pu+)))"); + EXPECT_TRUE(re.ok()); + const std::map& have = re.CapturingGroupNames(); + std::map want; + want[3] = "G2"; + want[6] = "G2"; + want[7] = "G1"; + EXPECT_EQ(want, have); +} + +TEST(RE2, RegexpToStringLossOfAnchor) { + EXPECT_EQ(RE2("^[a-c]at", RE2::POSIX).Regexp()->ToString(), "^[a-c]at"); + EXPECT_EQ(RE2("^[a-c]at").Regexp()->ToString(), "(?-m:^)[a-c]at"); + EXPECT_EQ(RE2("ca[t-z]$", RE2::POSIX).Regexp()->ToString(), "ca[t-z]$"); + EXPECT_EQ(RE2("ca[t-z]$").Regexp()->ToString(), "ca[t-z](?-m:$)"); +} + +// Issue 10131674 +TEST(RE2, Bug10131674) { + // Some of these escapes describe values that do not fit in a byte. + RE2 re("\\140\\440\\174\\271\\150\\656\\106\\201\\004\\332", RE2::Latin1); + EXPECT_FALSE(re.ok()); + EXPECT_FALSE(RE2::FullMatch("hello world", re)); +} + +TEST(RE2, Bug18391750) { + // Stray write past end of match_ in nfa.cc, caught by fuzzing + address sanitizer. + const char t[] = { + (char)0x28, (char)0x28, (char)0xfc, (char)0xfc, (char)0x08, (char)0x08, + (char)0x26, (char)0x26, (char)0x28, (char)0xc2, (char)0x9b, (char)0xc5, + (char)0xc5, (char)0xd4, (char)0x8f, (char)0x8f, (char)0x69, (char)0x69, + (char)0xe7, (char)0x29, (char)0x7b, (char)0x37, (char)0x31, (char)0x31, + (char)0x7d, (char)0xae, (char)0x7c, (char)0x7c, (char)0xf3, (char)0x29, + (char)0xae, (char)0xae, (char)0x2e, (char)0x2a, (char)0x29, (char)0x00, + }; + RE2::Options opt; + opt.set_encoding(RE2::Options::EncodingLatin1); + opt.set_longest_match(true); + opt.set_dot_nl(true); + opt.set_case_sensitive(false); + RE2 re(t, opt); + ASSERT_TRUE(re.ok()); + RE2::PartialMatch(t, re); +} + +TEST(RE2, Bug18458852) { + // Bug in parser accepting invalid (too large) rune, + // causing compiler to fail in DCHECK in UTF-8 + // character class code. + const char b[] = { + (char)0x28, (char)0x05, (char)0x05, (char)0x41, (char)0x41, (char)0x28, + (char)0x24, (char)0x5b, (char)0x5e, (char)0xf5, (char)0x87, (char)0x87, + (char)0x90, (char)0x29, (char)0x5d, (char)0x29, (char)0x29, (char)0x00, + }; + RE2 re(b); + ASSERT_FALSE(re.ok()); +} + +TEST(RE2, Bug18523943) { + // Bug in BitState: case kFailInst failed the match entirely. + + RE2::Options opt; + const char a[] = { + (char)0x29, (char)0x29, (char)0x24, (char)0x00, + }; + const char b[] = { + (char)0x28, (char)0x0a, (char)0x2a, (char)0x2a, (char)0x29, (char)0x00, + }; + opt.set_log_errors(false); + opt.set_encoding(RE2::Options::EncodingLatin1); + opt.set_posix_syntax(true); + opt.set_longest_match(true); + opt.set_literal(false); + opt.set_never_nl(true); + + RE2 re((const char*)b, opt); + ASSERT_TRUE(re.ok()); + std::string s1; + ASSERT_TRUE(RE2::PartialMatch((const char*)a, re, &s1)); +} + +TEST(RE2, Bug21371806) { + // Bug in parser accepting Unicode groups in Latin-1 mode, + // causing compiler to fail in DCHECK in prog.cc. + + RE2::Options opt; + opt.set_encoding(RE2::Options::EncodingLatin1); + + RE2 re("g\\p{Zl}]", opt); + ASSERT_TRUE(re.ok()); +} + +TEST(RE2, Bug26356109) { + // Bug in parser caused by factoring of common prefixes in alternations. + + // In the past, this was factored to "a\\C*?[bc]". Thus, the automaton would + // consume "ab" and then stop (when unanchored) whereas it should consume all + // of "abc" as per first-match semantics. + RE2 re("a\\C*?c|a\\C*?b"); + ASSERT_TRUE(re.ok()); + + std::string s = "abc"; + StringPiece m; + + ASSERT_TRUE(re.Match(s, 0, s.size(), RE2::UNANCHORED, &m, 1)); + ASSERT_EQ(m, s) << " (UNANCHORED) got m='" << m << "', want '" << s << "'"; + + ASSERT_TRUE(re.Match(s, 0, s.size(), RE2::ANCHOR_BOTH, &m, 1)); + ASSERT_EQ(m, s) << " (ANCHOR_BOTH) got m='" << m << "', want '" << s << "'"; +} + +TEST(RE2, Issue104) { + // RE2::GlobalReplace always advanced by one byte when the empty string was + // matched, which would clobber any rune that is longer than one byte. + + std::string s = "bc"; + ASSERT_EQ(3, RE2::GlobalReplace(&s, "a*", "d")); + ASSERT_EQ("dbdcd", s); + + s = "ąć"; + ASSERT_EQ(3, RE2::GlobalReplace(&s, "Ć*", "Ĉ")); + ASSERT_EQ("ĈąĈćĈ", s); + + s = "人类"; + ASSERT_EQ(3, RE2::GlobalReplace(&s, "大*", "小")); + ASSERT_EQ("小人小类小", s); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/regexp_benchmark.cc b/extern/re2/re2/testing/regexp_benchmark.cc new file mode 100644 index 0000000000..968fb86a64 --- /dev/null +++ b/extern/re2/re2/testing/regexp_benchmark.cc @@ -0,0 +1,1586 @@ +// Copyright 2006-2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Benchmarks for regular expression implementations. + +#include +#include +#include +#include +#include + +#include "util/test.h" +#include "util/logging.h" +#include "util/strutil.h" +#include "re2/prog.h" +#include "re2/re2.h" +#include "re2/regexp.h" +#include "util/pcre.h" +#include "util/benchmark.h" + +namespace re2 { +void Test(); +void MemoryUsage(); +} // namespace re2 + +typedef testing::MallocCounter MallocCounter; + +namespace re2 { + +void Test() { + Regexp* re = Regexp::Parse("(\\d+)-(\\d+)-(\\d+)", Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->IsOnePass()); + CHECK(prog->CanBitState()); + const char* text = "650-253-0001"; + StringPiece sp[4]; + CHECK(prog->SearchOnePass(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 4)); + CHECK_EQ(sp[0], "650-253-0001"); + CHECK_EQ(sp[1], "650"); + CHECK_EQ(sp[2], "253"); + CHECK_EQ(sp[3], "0001"); + delete prog; + re->Decref(); + LOG(INFO) << "test passed\n"; +} + +void MemoryUsage() { + const char* regexp = "(\\d+)-(\\d+)-(\\d+)"; + const char* text = "650-253-0001"; + { + MallocCounter mc(MallocCounter::THIS_THREAD_ONLY); + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + // Can't pass mc.HeapGrowth() and mc.PeakHeapGrowth() to LOG(INFO) directly, + // because LOG(INFO) might do a big allocation before they get evaluated. + fprintf(stderr, "Regexp: %7lld bytes (peak=%lld)\n", mc.HeapGrowth(), mc.PeakHeapGrowth()); + mc.Reset(); + + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->IsOnePass()); + CHECK(prog->CanBitState()); + fprintf(stderr, "Prog: %7lld bytes (peak=%lld)\n", mc.HeapGrowth(), mc.PeakHeapGrowth()); + mc.Reset(); + + StringPiece sp[4]; + CHECK(prog->SearchOnePass(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 4)); + fprintf(stderr, "Search: %7lld bytes (peak=%lld)\n", mc.HeapGrowth(), mc.PeakHeapGrowth()); + delete prog; + re->Decref(); + } + + { + MallocCounter mc(MallocCounter::THIS_THREAD_ONLY); + + PCRE re(regexp, PCRE::UTF8); + fprintf(stderr, "RE: %7lld bytes (peak=%lld)\n", mc.HeapGrowth(), mc.PeakHeapGrowth()); + PCRE::FullMatch(text, re); + fprintf(stderr, "RE: %7lld bytes (peak=%lld)\n", mc.HeapGrowth(), mc.PeakHeapGrowth()); + } + + { + MallocCounter mc(MallocCounter::THIS_THREAD_ONLY); + + PCRE* re = new PCRE(regexp, PCRE::UTF8); + fprintf(stderr, "PCRE*: %7lld bytes (peak=%lld)\n", mc.HeapGrowth(), mc.PeakHeapGrowth()); + PCRE::FullMatch(text, *re); + fprintf(stderr, "PCRE*: %7lld bytes (peak=%lld)\n", mc.HeapGrowth(), mc.PeakHeapGrowth()); + delete re; + } + + { + MallocCounter mc(MallocCounter::THIS_THREAD_ONLY); + + RE2 re(regexp); + fprintf(stderr, "RE2: %7lld bytes (peak=%lld)\n", mc.HeapGrowth(), mc.PeakHeapGrowth()); + RE2::FullMatch(text, re); + fprintf(stderr, "RE2: %7lld bytes (peak=%lld)\n", mc.HeapGrowth(), mc.PeakHeapGrowth()); + } + + fprintf(stderr, "sizeof: PCRE=%zd RE2=%zd Prog=%zd Inst=%zd\n", + sizeof(PCRE), sizeof(RE2), sizeof(Prog), sizeof(Prog::Inst)); +} + +// Regular expression implementation wrappers. +// Defined at bottom of file, but they are repetitive +// and not interesting. + +typedef void SearchImpl(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match); + +SearchImpl SearchDFA, SearchNFA, SearchOnePass, SearchBitState, + SearchPCRE, SearchRE2, + SearchCachedDFA, SearchCachedNFA, SearchCachedOnePass, SearchCachedBitState, + SearchCachedPCRE, SearchCachedRE2; + +typedef void ParseImpl(int iters, const char* regexp, const StringPiece& text); + +ParseImpl Parse1NFA, Parse1OnePass, Parse1BitState, + Parse1PCRE, Parse1RE2, + Parse1Backtrack, + Parse1CachedNFA, Parse1CachedOnePass, Parse1CachedBitState, + Parse1CachedPCRE, Parse1CachedRE2, + Parse1CachedBacktrack; + +ParseImpl Parse3NFA, Parse3OnePass, Parse3BitState, + Parse3PCRE, Parse3RE2, + Parse3Backtrack, + Parse3CachedNFA, Parse3CachedOnePass, Parse3CachedBitState, + Parse3CachedPCRE, Parse3CachedRE2, + Parse3CachedBacktrack; + +ParseImpl SearchParse2CachedPCRE, SearchParse2CachedRE2; + +ParseImpl SearchParse1CachedPCRE, SearchParse1CachedRE2; + +// Benchmark: failed search for regexp in random text. + +// Generate random text that won't contain the search string, +// to test worst-case search behavior. +void MakeText(std::string* text, int nbytes) { + srand(1); + text->resize(nbytes); + for (int i = 0; i < nbytes; i++) { + // Generate a one-byte rune that isn't a control character (e.g. '\n'). + // Clipping to 0x20 introduces some bias, but we don't need uniformity. + int byte = rand() & 0x7F; + if (byte < 0x20) + byte = 0x20; + (*text)[i] = byte; + } +} + +// Makes text of size nbytes, then calls run to search +// the text for regexp iters times. +void Search(int iters, int nbytes, const char* regexp, SearchImpl* search) { + StopBenchmarkTiming(); + std::string s; + MakeText(&s, nbytes); + BenchmarkMemoryUsage(); + StartBenchmarkTiming(); + search(iters, regexp, s, Prog::kUnanchored, false); + SetBenchmarkBytesProcessed(static_cast(iters)*nbytes); +} + +// These two are easy because they start with an A, +// giving the search loop something to memchr for. +#define EASY0 "ABCDEFGHIJKLMNOPQRSTUVWXYZ$" +#define EASY1 "A[AB]B[BC]C[CD]D[DE]E[EF]F[FG]G[GH]H[HI]I[IJ]J$" + +// This is a little harder, since it starts with a character class +// and thus can't be memchr'ed. Could look for ABC and work backward, +// but no one does that. +#define MEDIUM "[XYZ]ABCDEFGHIJKLMNOPQRSTUVWXYZ$" + +// This is a fair amount harder, because of the leading [ -~]*. +// A bad backtracking implementation will take O(text^2) time to +// figure out there's no match. +#define HARD "[ -~]*ABCDEFGHIJKLMNOPQRSTUVWXYZ$" + +// This has quite a high degree of fanout. +// NFA execution will be particularly slow. +#define FANOUT "(?:[\\x{80}-\\x{10FFFF}]?){100}[\\x{80}-\\x{10FFFF}]" + +// This stresses engines that are trying to track parentheses. +#define PARENS "([ -~])*(A)(B)(C)(D)(E)(F)(G)(H)(I)(J)(K)(L)(M)" \ + "(N)(O)(P)(Q)(R)(S)(T)(U)(V)(W)(X)(Y)(Z)$" + +void Search_Easy0_CachedDFA(int i, int n) { Search(i, n, EASY0, SearchCachedDFA); } +void Search_Easy0_CachedNFA(int i, int n) { Search(i, n, EASY0, SearchCachedNFA); } +void Search_Easy0_CachedPCRE(int i, int n) { Search(i, n, EASY0, SearchCachedPCRE); } +void Search_Easy0_CachedRE2(int i, int n) { Search(i, n, EASY0, SearchCachedRE2); } + +BENCHMARK_RANGE(Search_Easy0_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Easy0_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_Easy0_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_Easy0_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); + +void Search_Easy1_CachedDFA(int i, int n) { Search(i, n, EASY1, SearchCachedDFA); } +void Search_Easy1_CachedNFA(int i, int n) { Search(i, n, EASY1, SearchCachedNFA); } +void Search_Easy1_CachedPCRE(int i, int n) { Search(i, n, EASY1, SearchCachedPCRE); } +void Search_Easy1_CachedRE2(int i, int n) { Search(i, n, EASY1, SearchCachedRE2); } + +BENCHMARK_RANGE(Search_Easy1_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Easy1_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_Easy1_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_Easy1_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); + +void Search_Medium_CachedDFA(int i, int n) { Search(i, n, MEDIUM, SearchCachedDFA); } +void Search_Medium_CachedNFA(int i, int n) { Search(i, n, MEDIUM, SearchCachedNFA); } +void Search_Medium_CachedPCRE(int i, int n) { Search(i, n, MEDIUM, SearchCachedPCRE); } +void Search_Medium_CachedRE2(int i, int n) { Search(i, n, MEDIUM, SearchCachedRE2); } + +BENCHMARK_RANGE(Search_Medium_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Medium_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_Medium_CachedPCRE, 8, 256<<10)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_Medium_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); + +void Search_Hard_CachedDFA(int i, int n) { Search(i, n, HARD, SearchCachedDFA); } +void Search_Hard_CachedNFA(int i, int n) { Search(i, n, HARD, SearchCachedNFA); } +void Search_Hard_CachedPCRE(int i, int n) { Search(i, n, HARD, SearchCachedPCRE); } +void Search_Hard_CachedRE2(int i, int n) { Search(i, n, HARD, SearchCachedRE2); } + +BENCHMARK_RANGE(Search_Hard_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Hard_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_Hard_CachedPCRE, 8, 4<<10)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_Hard_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); + +void Search_Fanout_CachedDFA(int i, int n) { Search(i, n, FANOUT, SearchCachedDFA); } +void Search_Fanout_CachedNFA(int i, int n) { Search(i, n, FANOUT, SearchCachedNFA); } +void Search_Fanout_CachedPCRE(int i, int n) { Search(i, n, FANOUT, SearchCachedPCRE); } +void Search_Fanout_CachedRE2(int i, int n) { Search(i, n, FANOUT, SearchCachedRE2); } + +BENCHMARK_RANGE(Search_Fanout_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Fanout_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_Fanout_CachedPCRE, 8, 4<<10)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_Fanout_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); + +void Search_Parens_CachedDFA(int i, int n) { Search(i, n, PARENS, SearchCachedDFA); } +void Search_Parens_CachedNFA(int i, int n) { Search(i, n, PARENS, SearchCachedNFA); } +void Search_Parens_CachedPCRE(int i, int n) { Search(i, n, PARENS, SearchCachedPCRE); } +void Search_Parens_CachedRE2(int i, int n) { Search(i, n, PARENS, SearchCachedRE2); } + +BENCHMARK_RANGE(Search_Parens_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Parens_CachedNFA, 8, 256<<10)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_Parens_CachedPCRE, 8, 8)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_Parens_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); + +void SearchBigFixed(int iters, int nbytes, SearchImpl* search) { + StopBenchmarkTiming(); + std::string s; + s.append(nbytes/2, 'x'); + std::string regexp = "^" + s + ".*$"; + std::string t; + MakeText(&t, nbytes/2); + s += t; + BenchmarkMemoryUsage(); + StartBenchmarkTiming(); + search(iters, regexp.c_str(), s, Prog::kUnanchored, true); + SetBenchmarkBytesProcessed(static_cast(iters)*nbytes); +} + +void Search_BigFixed_CachedDFA(int i, int n) { SearchBigFixed(i, n, SearchCachedDFA); } +void Search_BigFixed_CachedNFA(int i, int n) { SearchBigFixed(i, n, SearchCachedNFA); } +void Search_BigFixed_CachedPCRE(int i, int n) { SearchBigFixed(i, n, SearchCachedPCRE); } +void Search_BigFixed_CachedRE2(int i, int n) { SearchBigFixed(i, n, SearchCachedRE2); } + +BENCHMARK_RANGE(Search_BigFixed_CachedDFA, 8, 1<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_BigFixed_CachedNFA, 8, 32<<10)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_BigFixed_CachedPCRE, 8, 32<<10)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_BigFixed_CachedRE2, 8, 1<<20)->ThreadRange(1, NumCPUs()); + +// Benchmark: FindAndConsume + +void FindAndConsume(int iters, int nbytes) { + StopBenchmarkTiming(); + std::string s; + MakeText(&s, nbytes); + s.append("Hello World"); + StartBenchmarkTiming(); + RE2 re("((Hello World))"); + for (int i = 0; i < iters; i++) { + StringPiece t = s; + StringPiece u; + CHECK(RE2::FindAndConsume(&t, re, &u)); + CHECK_EQ(u, "Hello World"); + } + SetBenchmarkBytesProcessed(static_cast(iters)*nbytes); +} + +BENCHMARK_RANGE(FindAndConsume, 8, 16<<20)->ThreadRange(1, NumCPUs()); + +// Benchmark: successful anchored search. + +void SearchSuccess(int iters, int nbytes, const char* regexp, SearchImpl* search) { + StopBenchmarkTiming(); + std::string s; + MakeText(&s, nbytes); + BenchmarkMemoryUsage(); + StartBenchmarkTiming(); + search(iters, regexp, s, Prog::kAnchored, true); + SetBenchmarkBytesProcessed(static_cast(iters)*nbytes); +} + +// Unambiguous search (RE2 can use OnePass). + +void Search_Success_DFA(int i, int n) { SearchSuccess(i, n, ".*$", SearchDFA); } +void Search_Success_NFA(int i, int n) { SearchSuccess(i, n, ".*$", SearchNFA); } +void Search_Success_PCRE(int i, int n) { SearchSuccess(i, n, ".*$", SearchPCRE); } +void Search_Success_RE2(int i, int n) { SearchSuccess(i, n, ".*$", SearchRE2); } +void Search_Success_OnePass(int i, int n) { SearchSuccess(i, n, ".*$", SearchOnePass); } + +BENCHMARK_RANGE(Search_Success_DFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Success_NFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_Success_PCRE, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_Success_RE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Success_OnePass, 8, 2<<20)->ThreadRange(1, NumCPUs()); + +void Search_Success_CachedDFA(int i, int n) { SearchSuccess(i, n, ".*$", SearchCachedDFA); } +void Search_Success_CachedNFA(int i, int n) { SearchSuccess(i, n, ".*$", SearchCachedNFA); } +void Search_Success_CachedPCRE(int i, int n) { SearchSuccess(i, n, ".*$", SearchCachedPCRE); } +void Search_Success_CachedRE2(int i, int n) { SearchSuccess(i, n, ".*$", SearchCachedRE2); } +void Search_Success_CachedOnePass(int i, int n) { SearchSuccess(i, n, ".*$", SearchCachedOnePass); } + +BENCHMARK_RANGE(Search_Success_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Success_CachedNFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_Success_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_Success_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Success_CachedOnePass, 8, 2<<20)->ThreadRange(1, NumCPUs()); + +// Ambiguous search (RE2 cannot use OnePass). +// Used to be ".*.$", but that is coalesced to ".+$" these days. + +void Search_Success1_DFA(int i, int n) { SearchSuccess(i, n, ".*\\C$", SearchDFA); } +void Search_Success1_NFA(int i, int n) { SearchSuccess(i, n, ".*\\C$", SearchNFA); } +void Search_Success1_PCRE(int i, int n) { SearchSuccess(i, n, ".*\\C$", SearchPCRE); } +void Search_Success1_RE2(int i, int n) { SearchSuccess(i, n, ".*\\C$", SearchRE2); } +void Search_Success1_BitState(int i, int n) { SearchSuccess(i, n, ".*\\C$", SearchBitState); } + +BENCHMARK_RANGE(Search_Success1_DFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Success1_NFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_Success1_PCRE, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_Success1_RE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Success1_BitState, 8, 2<<20)->ThreadRange(1, NumCPUs()); + +void Search_Success1_CachedDFA(int i, int n) { SearchSuccess(i, n, ".*\\C$", SearchCachedDFA); } +void Search_Success1_CachedNFA(int i, int n) { SearchSuccess(i, n, ".*\\C$", SearchCachedNFA); } +void Search_Success1_CachedPCRE(int i, int n) { SearchSuccess(i, n, ".*\\C$", SearchCachedPCRE); } +void Search_Success1_CachedRE2(int i, int n) { SearchSuccess(i, n, ".*\\C$", SearchCachedRE2); } +void Search_Success1_CachedBitState(int i, int n) { SearchSuccess(i, n, ".*\\C$", SearchCachedBitState); } + +BENCHMARK_RANGE(Search_Success1_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Success1_CachedNFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_Success1_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_Success1_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_Success1_CachedBitState, 8, 2<<20)->ThreadRange(1, NumCPUs()); + +// Benchmark: AltMatch optimisation (just to verify that it works) +// Note that OnePass doesn't implement it! + +void SearchAltMatch(int iters, int nbytes, SearchImpl* search) { + StopBenchmarkTiming(); + std::string s; + MakeText(&s, nbytes); + BenchmarkMemoryUsage(); + StartBenchmarkTiming(); + search(iters, "\\C*", s, Prog::kAnchored, true); + SetBenchmarkBytesProcessed(static_cast(iters)*nbytes); +} + +void Search_AltMatch_DFA(int i, int n) { SearchAltMatch(i, n, SearchDFA); } +void Search_AltMatch_NFA(int i, int n) { SearchAltMatch(i, n, SearchNFA); } +void Search_AltMatch_OnePass(int i, int n) { SearchAltMatch(i, n, SearchOnePass); } +void Search_AltMatch_BitState(int i, int n) { SearchAltMatch(i, n, SearchBitState); } +void Search_AltMatch_PCRE(int i, int n) { SearchAltMatch(i, n, SearchPCRE); } +void Search_AltMatch_RE2(int i, int n) { SearchAltMatch(i, n, SearchRE2); } + +BENCHMARK_RANGE(Search_AltMatch_DFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_AltMatch_NFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_AltMatch_OnePass, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_AltMatch_BitState, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_AltMatch_PCRE, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_AltMatch_RE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); + +void Search_AltMatch_CachedDFA(int i, int n) { SearchAltMatch(i, n, SearchCachedDFA); } +void Search_AltMatch_CachedNFA(int i, int n) { SearchAltMatch(i, n, SearchCachedNFA); } +void Search_AltMatch_CachedOnePass(int i, int n) { SearchAltMatch(i, n, SearchCachedOnePass); } +void Search_AltMatch_CachedBitState(int i, int n) { SearchAltMatch(i, n, SearchCachedBitState); } +void Search_AltMatch_CachedPCRE(int i, int n) { SearchAltMatch(i, n, SearchCachedPCRE); } +void Search_AltMatch_CachedRE2(int i, int n) { SearchAltMatch(i, n, SearchCachedRE2); } + +BENCHMARK_RANGE(Search_AltMatch_CachedDFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_AltMatch_CachedNFA, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_AltMatch_CachedOnePass, 8, 16<<20)->ThreadRange(1, NumCPUs()); +BENCHMARK_RANGE(Search_AltMatch_CachedBitState, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK_RANGE(Search_AltMatch_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(Search_AltMatch_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); + +// Benchmark: use regexp to find phone number. + +void SearchDigits(int iters, SearchImpl* search) { + StringPiece s("650-253-0001"); + BenchmarkMemoryUsage(); + search(iters, "([0-9]+)-([0-9]+)-([0-9]+)", s, Prog::kAnchored, true); + SetBenchmarkItemsProcessed(iters); +} + +void Search_Digits_DFA(int i) { SearchDigits(i, SearchDFA); } +void Search_Digits_NFA(int i) { SearchDigits(i, SearchNFA); } +void Search_Digits_OnePass(int i) { SearchDigits(i, SearchOnePass); } +void Search_Digits_PCRE(int i) { SearchDigits(i, SearchPCRE); } +void Search_Digits_RE2(int i) { SearchDigits(i, SearchRE2); } +void Search_Digits_BitState(int i) { SearchDigits(i, SearchBitState); } + +BENCHMARK(Search_Digits_DFA)->ThreadRange(1, NumCPUs()); +BENCHMARK(Search_Digits_NFA)->ThreadRange(1, NumCPUs()); +BENCHMARK(Search_Digits_OnePass)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK(Search_Digits_PCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(Search_Digits_RE2)->ThreadRange(1, NumCPUs()); +BENCHMARK(Search_Digits_BitState)->ThreadRange(1, NumCPUs()); + +// Benchmark: use regexp to parse digit fields in phone number. + +void Parse3Digits(int iters, + void (*parse3)(int, const char*, const StringPiece&)) { + BenchmarkMemoryUsage(); + parse3(iters, "([0-9]+)-([0-9]+)-([0-9]+)", "650-253-0001"); + SetBenchmarkItemsProcessed(iters); +} + +void Parse_Digits_NFA(int i) { Parse3Digits(i, Parse3NFA); } +void Parse_Digits_OnePass(int i) { Parse3Digits(i, Parse3OnePass); } +void Parse_Digits_PCRE(int i) { Parse3Digits(i, Parse3PCRE); } +void Parse_Digits_RE2(int i) { Parse3Digits(i, Parse3RE2); } +void Parse_Digits_Backtrack(int i) { Parse3Digits(i, Parse3Backtrack); } +void Parse_Digits_BitState(int i) { Parse3Digits(i, Parse3BitState); } + +BENCHMARK(Parse_Digits_NFA)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_Digits_OnePass)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK(Parse_Digits_PCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(Parse_Digits_RE2)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_Digits_Backtrack)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_Digits_BitState)->ThreadRange(1, NumCPUs()); + +void Parse_CachedDigits_NFA(int i) { Parse3Digits(i, Parse3CachedNFA); } +void Parse_CachedDigits_OnePass(int i) { Parse3Digits(i, Parse3CachedOnePass); } +void Parse_CachedDigits_PCRE(int i) { Parse3Digits(i, Parse3CachedPCRE); } +void Parse_CachedDigits_RE2(int i) { Parse3Digits(i, Parse3CachedRE2); } +void Parse_CachedDigits_Backtrack(int i) { Parse3Digits(i, Parse3CachedBacktrack); } +void Parse_CachedDigits_BitState(int i) { Parse3Digits(i, Parse3CachedBitState); } + +BENCHMARK(Parse_CachedDigits_NFA)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_CachedDigits_OnePass)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK(Parse_CachedDigits_PCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(Parse_CachedDigits_Backtrack)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_CachedDigits_RE2)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_CachedDigits_BitState)->ThreadRange(1, NumCPUs()); + +void Parse3DigitDs(int iters, + void (*parse3)(int, const char*, const StringPiece&)) { + BenchmarkMemoryUsage(); + parse3(iters, "(\\d+)-(\\d+)-(\\d+)", "650-253-0001"); + SetBenchmarkItemsProcessed(iters); +} + +void Parse_DigitDs_NFA(int i) { Parse3DigitDs(i, Parse3NFA); } +void Parse_DigitDs_OnePass(int i) { Parse3DigitDs(i, Parse3OnePass); } +void Parse_DigitDs_PCRE(int i) { Parse3DigitDs(i, Parse3PCRE); } +void Parse_DigitDs_RE2(int i) { Parse3DigitDs(i, Parse3RE2); } +void Parse_DigitDs_Backtrack(int i) { Parse3DigitDs(i, Parse3CachedBacktrack); } +void Parse_DigitDs_BitState(int i) { Parse3DigitDs(i, Parse3CachedBitState); } + +BENCHMARK(Parse_DigitDs_NFA)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_DigitDs_OnePass)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK(Parse_DigitDs_PCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(Parse_DigitDs_RE2)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_DigitDs_Backtrack)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_DigitDs_BitState)->ThreadRange(1, NumCPUs()); + +void Parse_CachedDigitDs_NFA(int i) { Parse3DigitDs(i, Parse3CachedNFA); } +void Parse_CachedDigitDs_OnePass(int i) { Parse3DigitDs(i, Parse3CachedOnePass); } +void Parse_CachedDigitDs_PCRE(int i) { Parse3DigitDs(i, Parse3CachedPCRE); } +void Parse_CachedDigitDs_RE2(int i) { Parse3DigitDs(i, Parse3CachedRE2); } +void Parse_CachedDigitDs_Backtrack(int i) { Parse3DigitDs(i, Parse3CachedBacktrack); } +void Parse_CachedDigitDs_BitState(int i) { Parse3DigitDs(i, Parse3CachedBitState); } + +BENCHMARK(Parse_CachedDigitDs_NFA)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_CachedDigitDs_OnePass)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK(Parse_CachedDigitDs_PCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(Parse_CachedDigitDs_Backtrack)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_CachedDigitDs_RE2)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_CachedDigitDs_BitState)->ThreadRange(1, NumCPUs()); + +// Benchmark: splitting off leading number field. + +void Parse1Split(int iters, + void (*parse1)(int, const char*, const StringPiece&)) { + BenchmarkMemoryUsage(); + parse1(iters, "[0-9]+-(.*)", "650-253-0001"); + SetBenchmarkItemsProcessed(iters); +} + +void Parse_Split_NFA(int i) { Parse1Split(i, Parse1NFA); } +void Parse_Split_OnePass(int i) { Parse1Split(i, Parse1OnePass); } +void Parse_Split_PCRE(int i) { Parse1Split(i, Parse1PCRE); } +void Parse_Split_RE2(int i) { Parse1Split(i, Parse1RE2); } +void Parse_Split_BitState(int i) { Parse1Split(i, Parse1BitState); } + +BENCHMARK(Parse_Split_NFA)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_Split_OnePass)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK(Parse_Split_PCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(Parse_Split_RE2)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_Split_BitState)->ThreadRange(1, NumCPUs()); + +void Parse_CachedSplit_NFA(int i) { Parse1Split(i, Parse1CachedNFA); } +void Parse_CachedSplit_OnePass(int i) { Parse1Split(i, Parse1CachedOnePass); } +void Parse_CachedSplit_PCRE(int i) { Parse1Split(i, Parse1CachedPCRE); } +void Parse_CachedSplit_RE2(int i) { Parse1Split(i, Parse1CachedRE2); } +void Parse_CachedSplit_BitState(int i) { Parse1Split(i, Parse1CachedBitState); } + +BENCHMARK(Parse_CachedSplit_NFA)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_CachedSplit_OnePass)->ThreadRange(1, NumCPUs()); +#ifdef USEPCRE +BENCHMARK(Parse_CachedSplit_PCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(Parse_CachedSplit_RE2)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_CachedSplit_BitState)->ThreadRange(1, NumCPUs()); + +// Benchmark: splitting off leading number field but harder (ambiguous regexp). + +void Parse1SplitHard(int iters, + void (*run)(int, const char*, const StringPiece&)) { + BenchmarkMemoryUsage(); + run(iters, "[0-9]+.(.*)", "650-253-0001"); + SetBenchmarkItemsProcessed(iters); +} + +void Parse_SplitHard_NFA(int i) { Parse1SplitHard(i, Parse1NFA); } +void Parse_SplitHard_PCRE(int i) { Parse1SplitHard(i, Parse1PCRE); } +void Parse_SplitHard_RE2(int i) { Parse1SplitHard(i, Parse1RE2); } +void Parse_SplitHard_BitState(int i) { Parse1SplitHard(i, Parse1BitState); } + +#ifdef USEPCRE +BENCHMARK(Parse_SplitHard_PCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(Parse_SplitHard_RE2)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_SplitHard_BitState)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_SplitHard_NFA)->ThreadRange(1, NumCPUs()); + +void Parse_CachedSplitHard_NFA(int i) { Parse1SplitHard(i, Parse1CachedNFA); } +void Parse_CachedSplitHard_PCRE(int i) { Parse1SplitHard(i, Parse1CachedPCRE); } +void Parse_CachedSplitHard_RE2(int i) { Parse1SplitHard(i, Parse1CachedRE2); } +void Parse_CachedSplitHard_BitState(int i) { Parse1SplitHard(i, Parse1CachedBitState); } +void Parse_CachedSplitHard_Backtrack(int i) { Parse1SplitHard(i, Parse1CachedBacktrack); } + +#ifdef USEPCRE +BENCHMARK(Parse_CachedSplitHard_PCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(Parse_CachedSplitHard_RE2)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_CachedSplitHard_BitState)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_CachedSplitHard_NFA)->ThreadRange(1, NumCPUs()); +BENCHMARK(Parse_CachedSplitHard_Backtrack)->ThreadRange(1, NumCPUs()); + +// Benchmark: Parse1SplitHard, big text, small match. + +void Parse1SplitBig1(int iters, + void (*run)(int, const char*, const StringPiece&)) { + std::string s; + s.append(100000, 'x'); + s.append("650-253-0001"); + BenchmarkMemoryUsage(); + run(iters, "[0-9]+.(.*)", s); + SetBenchmarkItemsProcessed(iters); +} + +void Parse_CachedSplitBig1_PCRE(int i) { Parse1SplitBig1(i, SearchParse1CachedPCRE); } +void Parse_CachedSplitBig1_RE2(int i) { Parse1SplitBig1(i, SearchParse1CachedRE2); } + +#ifdef USEPCRE +BENCHMARK(Parse_CachedSplitBig1_PCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(Parse_CachedSplitBig1_RE2)->ThreadRange(1, NumCPUs()); + +// Benchmark: Parse1SplitHard, big text, big match. + +void Parse1SplitBig2(int iters, + void (*run)(int, const char*, const StringPiece&)) { + std::string s; + s.append("650-253-"); + s.append(100000, '0'); + BenchmarkMemoryUsage(); + run(iters, "[0-9]+.(.*)", s); + SetBenchmarkItemsProcessed(iters); +} + +void Parse_CachedSplitBig2_PCRE(int i) { Parse1SplitBig2(i, SearchParse1CachedPCRE); } +void Parse_CachedSplitBig2_RE2(int i) { Parse1SplitBig2(i, SearchParse1CachedRE2); } + +#ifdef USEPCRE +BENCHMARK(Parse_CachedSplitBig2_PCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(Parse_CachedSplitBig2_RE2)->ThreadRange(1, NumCPUs()); + +// Benchmark: measure time required to parse (but not execute) +// a simple regular expression. + +void ParseRegexp(int iters, const std::string& regexp) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + re->Decref(); + } +} + +void SimplifyRegexp(int iters, const std::string& regexp) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Regexp* sre = re->Simplify(); + CHECK(sre); + sre->Decref(); + re->Decref(); + } +} + +void NullWalkRegexp(int iters, const std::string& regexp) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + for (int i = 0; i < iters; i++) { + re->NullWalk(); + } + re->Decref(); +} + +void SimplifyCompileRegexp(int iters, const std::string& regexp) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Regexp* sre = re->Simplify(); + CHECK(sre); + Prog* prog = sre->CompileToProg(0); + CHECK(prog); + delete prog; + sre->Decref(); + re->Decref(); + } +} + +void CompileRegexp(int iters, const std::string& regexp) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + delete prog; + re->Decref(); + } +} + +void CompileToProg(int iters, const std::string& regexp) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + for (int i = 0; i < iters; i++) { + Prog* prog = re->CompileToProg(0); + CHECK(prog); + delete prog; + } + re->Decref(); +} + +void CompileByteMap(int iters, const std::string& regexp) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + for (int i = 0; i < iters; i++) { + prog->ComputeByteMap(); + } + delete prog; + re->Decref(); +} + +void CompilePCRE(int iters, const std::string& regexp) { + for (int i = 0; i < iters; i++) { + PCRE re(regexp, PCRE::UTF8); + CHECK_EQ(re.error(), ""); + } +} + +void CompileRE2(int iters, const std::string& regexp) { + for (int i = 0; i < iters; i++) { + RE2 re(regexp); + CHECK_EQ(re.error(), ""); + } +} + +void RunBuild(int iters, const std::string& regexp, + void (*run)(int, const std::string&)) { + run(iters, regexp); + SetBenchmarkItemsProcessed(iters); +} + +} // namespace re2 + +DEFINE_string(compile_regexp, "(.*)-(\\d+)-of-(\\d+)", "regexp for compile benchmarks"); + +namespace re2 { + +void BM_PCRE_Compile(int i) { RunBuild(i, FLAGS_compile_regexp, CompilePCRE); } +void BM_Regexp_Parse(int i) { RunBuild(i, FLAGS_compile_regexp, ParseRegexp); } +void BM_Regexp_Simplify(int i) { RunBuild(i, FLAGS_compile_regexp, SimplifyRegexp); } +void BM_CompileToProg(int i) { RunBuild(i, FLAGS_compile_regexp, CompileToProg); } +void BM_CompileByteMap(int i) { RunBuild(i, FLAGS_compile_regexp, CompileByteMap); } +void BM_Regexp_Compile(int i) { RunBuild(i, FLAGS_compile_regexp, CompileRegexp); } +void BM_Regexp_SimplifyCompile(int i) { RunBuild(i, FLAGS_compile_regexp, SimplifyCompileRegexp); } +void BM_Regexp_NullWalk(int i) { RunBuild(i, FLAGS_compile_regexp, NullWalkRegexp); } +void BM_RE2_Compile(int i) { RunBuild(i, FLAGS_compile_regexp, CompileRE2); } + +#ifdef USEPCRE +BENCHMARK(BM_PCRE_Compile)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(BM_Regexp_Parse)->ThreadRange(1, NumCPUs()); +BENCHMARK(BM_Regexp_Simplify)->ThreadRange(1, NumCPUs()); +BENCHMARK(BM_CompileToProg)->ThreadRange(1, NumCPUs()); +BENCHMARK(BM_CompileByteMap)->ThreadRange(1, NumCPUs()); +BENCHMARK(BM_Regexp_Compile)->ThreadRange(1, NumCPUs()); +BENCHMARK(BM_Regexp_SimplifyCompile)->ThreadRange(1, NumCPUs()); +BENCHMARK(BM_Regexp_NullWalk)->ThreadRange(1, NumCPUs()); +BENCHMARK(BM_RE2_Compile)->ThreadRange(1, NumCPUs()); + +// Makes text of size nbytes, then calls run to search +// the text for regexp iters times. +void SearchPhone(int iters, int nbytes, ParseImpl* search) { + StopBenchmarkTiming(); + std::string s; + MakeText(&s, nbytes); + s.append("(650) 253-0001"); + BenchmarkMemoryUsage(); + StartBenchmarkTiming(); + search(iters, "(\\d{3}-|\\(\\d{3}\\)\\s+)(\\d{3}-\\d{4})", s); + SetBenchmarkBytesProcessed(static_cast(iters)*nbytes); +} + +void SearchPhone_CachedPCRE(int i, int n) { + SearchPhone(i, n, SearchParse2CachedPCRE); +} +void SearchPhone_CachedRE2(int i, int n) { + SearchPhone(i, n, SearchParse2CachedRE2); +} + +#ifdef USEPCRE +BENCHMARK_RANGE(SearchPhone_CachedPCRE, 8, 16<<20)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK_RANGE(SearchPhone_CachedRE2, 8, 16<<20)->ThreadRange(1, NumCPUs()); + +/* +TODO(rsc): Make this work again. + +// Generates and returns a string over binary alphabet {0,1} that contains +// all possible binary sequences of length n as subsequences. The obvious +// brute force method would generate a string of length n * 2^n, but this +// generates a string of length n + 2^n - 1 called a De Bruijn cycle. +// See Knuth, The Art of Computer Programming, Vol 2, Exercise 3.2.2 #17. +static std::string DeBruijnString(int n) { + CHECK_LT(n, 8*sizeof(int)); + CHECK_GT(n, 0); + + std::vector did(1<(iters)*s.size()); +} + +void CacheFillPCRE(int i, int n) { CacheFill(i, n, SearchCachedPCRE); } +void CacheFillRE2(int i, int n) { CacheFill(i, n, SearchCachedRE2); } +void CacheFillNFA(int i, int n) { CacheFill(i, n, SearchCachedNFA); } +void CacheFillDFA(int i, int n) { CacheFill(i, n, SearchCachedDFA); } + +// BENCHMARK_WITH_ARG uses __LINE__ to generate distinct identifiers +// for the static BenchmarkRegisterer, which makes it unusable inside +// a macro like DO24 below. MY_BENCHMARK_WITH_ARG uses the argument a +// to make the identifiers distinct (only possible when 'a' is a simple +// expression like 2, not like 1+1). +#define MY_BENCHMARK_WITH_ARG(n, a) \ + bool __benchmark_ ## n ## a = \ + (new ::testing::Benchmark(#n, NewPermanentCallback(&n)))->ThreadRange(1, NumCPUs()); + +#define DO24(A, B) \ + A(B, 1); A(B, 2); A(B, 3); A(B, 4); A(B, 5); A(B, 6); \ + A(B, 7); A(B, 8); A(B, 9); A(B, 10); A(B, 11); A(B, 12); \ + A(B, 13); A(B, 14); A(B, 15); A(B, 16); A(B, 17); A(B, 18); \ + A(B, 19); A(B, 20); A(B, 21); A(B, 22); A(B, 23); A(B, 24); + +DO24(MY_BENCHMARK_WITH_ARG, CacheFillPCRE) +DO24(MY_BENCHMARK_WITH_ARG, CacheFillNFA) +DO24(MY_BENCHMARK_WITH_ARG, CacheFillRE2) +DO24(MY_BENCHMARK_WITH_ARG, CacheFillDFA) + +#undef DO24 +#undef MY_BENCHMARK_WITH_ARG +*/ + +//////////////////////////////////////////////////////////////////////// +// +// Implementation routines. Sad that there are so many, +// but all the interfaces are slightly different. + +// Runs implementation to search for regexp in text, iters times. +// Expect_match says whether the regexp should be found. +// Anchored says whether to run an anchored search. + +void SearchDFA(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + bool failed = false; + CHECK_EQ(prog->SearchDFA(text, StringPiece(), anchor, Prog::kFirstMatch, + NULL, &failed, NULL), + expect_match); + CHECK(!failed); + delete prog; + re->Decref(); + } +} + +void SearchNFA(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK_EQ(prog->SearchNFA(text, StringPiece(), anchor, Prog::kFirstMatch, + NULL, 0), + expect_match); + delete prog; + re->Decref(); + } +} + +void SearchOnePass(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->IsOnePass()); + CHECK_EQ(prog->SearchOnePass(text, text, anchor, Prog::kFirstMatch, NULL, 0), + expect_match); + delete prog; + re->Decref(); + } +} + +void SearchBitState(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->CanBitState()); + CHECK_EQ(prog->SearchBitState(text, text, anchor, Prog::kFirstMatch, NULL, 0), + expect_match); + delete prog; + re->Decref(); + } +} + +void SearchPCRE(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match) { + for (int i = 0; i < iters; i++) { + PCRE re(regexp, PCRE::UTF8); + CHECK_EQ(re.error(), ""); + if (anchor == Prog::kAnchored) + CHECK_EQ(PCRE::FullMatch(text, re), expect_match); + else + CHECK_EQ(PCRE::PartialMatch(text, re), expect_match); + } +} + +void SearchRE2(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match) { + for (int i = 0; i < iters; i++) { + RE2 re(regexp); + CHECK_EQ(re.error(), ""); + if (anchor == Prog::kAnchored) + CHECK_EQ(RE2::FullMatch(text, re), expect_match); + else + CHECK_EQ(RE2::PartialMatch(text, re), expect_match); + } +} + +// SearchCachedXXX is like SearchXXX but only does the +// regexp parsing and compiling once. This lets us measure +// search time without the per-regexp overhead. + +void SearchCachedDFA(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(1LL<<31); + CHECK(prog); + for (int i = 0; i < iters; i++) { + bool failed = false; + CHECK_EQ(prog->SearchDFA(text, StringPiece(), anchor, Prog::kFirstMatch, + NULL, &failed, NULL), + expect_match); + CHECK(!failed); + } + delete prog; + re->Decref(); +} + +void SearchCachedNFA(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + for (int i = 0; i < iters; i++) { + CHECK_EQ(prog->SearchNFA(text, StringPiece(), anchor, Prog::kFirstMatch, + NULL, 0), + expect_match); + } + delete prog; + re->Decref(); +} + +void SearchCachedOnePass(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->IsOnePass()); + for (int i = 0; i < iters; i++) + CHECK_EQ(prog->SearchOnePass(text, text, anchor, Prog::kFirstMatch, NULL, 0), + expect_match); + delete prog; + re->Decref(); +} + +void SearchCachedBitState(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->CanBitState()); + for (int i = 0; i < iters; i++) + CHECK_EQ(prog->SearchBitState(text, text, anchor, Prog::kFirstMatch, NULL, 0), + expect_match); + delete prog; + re->Decref(); +} + +void SearchCachedPCRE(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match) { + PCRE re(regexp, PCRE::UTF8); + CHECK_EQ(re.error(), ""); + for (int i = 0; i < iters; i++) { + if (anchor == Prog::kAnchored) + CHECK_EQ(PCRE::FullMatch(text, re), expect_match); + else + CHECK_EQ(PCRE::PartialMatch(text, re), expect_match); + } +} + +void SearchCachedRE2(int iters, const char* regexp, const StringPiece& text, + Prog::Anchor anchor, bool expect_match) { + RE2 re(regexp); + CHECK_EQ(re.error(), ""); + for (int i = 0; i < iters; i++) { + if (anchor == Prog::kAnchored) + CHECK_EQ(RE2::FullMatch(text, re), expect_match); + else + CHECK_EQ(RE2::PartialMatch(text, re), expect_match); + } +} + + +// Runs implementation to full match regexp against text, +// extracting three submatches. Expects match always. + +void Parse3NFA(int iters, const char* regexp, const StringPiece& text) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + StringPiece sp[4]; // 4 because sp[0] is whole match. + CHECK(prog->SearchNFA(text, StringPiece(), Prog::kAnchored, + Prog::kFullMatch, sp, 4)); + delete prog; + re->Decref(); + } +} + +void Parse3OnePass(int iters, const char* regexp, const StringPiece& text) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->IsOnePass()); + StringPiece sp[4]; // 4 because sp[0] is whole match. + CHECK(prog->SearchOnePass(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 4)); + delete prog; + re->Decref(); + } +} + +void Parse3BitState(int iters, const char* regexp, const StringPiece& text) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->CanBitState()); + StringPiece sp[4]; // 4 because sp[0] is whole match. + CHECK(prog->SearchBitState(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 4)); + delete prog; + re->Decref(); + } +} + +void Parse3Backtrack(int iters, const char* regexp, const StringPiece& text) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + StringPiece sp[4]; // 4 because sp[0] is whole match. + CHECK(prog->UnsafeSearchBacktrack(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 4)); + delete prog; + re->Decref(); + } +} + +void Parse3PCRE(int iters, const char* regexp, const StringPiece& text) { + for (int i = 0; i < iters; i++) { + PCRE re(regexp, PCRE::UTF8); + CHECK_EQ(re.error(), ""); + StringPiece sp1, sp2, sp3; + CHECK(PCRE::FullMatch(text, re, &sp1, &sp2, &sp3)); + } +} + +void Parse3RE2(int iters, const char* regexp, const StringPiece& text) { + for (int i = 0; i < iters; i++) { + RE2 re(regexp); + CHECK_EQ(re.error(), ""); + StringPiece sp1, sp2, sp3; + CHECK(RE2::FullMatch(text, re, &sp1, &sp2, &sp3)); + } +} + +void Parse3CachedNFA(int iters, const char* regexp, const StringPiece& text) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + StringPiece sp[4]; // 4 because sp[0] is whole match. + for (int i = 0; i < iters; i++) { + CHECK(prog->SearchNFA(text, StringPiece(), Prog::kAnchored, + Prog::kFullMatch, sp, 4)); + } + delete prog; + re->Decref(); +} + +void Parse3CachedOnePass(int iters, const char* regexp, const StringPiece& text) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->IsOnePass()); + StringPiece sp[4]; // 4 because sp[0] is whole match. + for (int i = 0; i < iters; i++) + CHECK(prog->SearchOnePass(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 4)); + delete prog; + re->Decref(); +} + +void Parse3CachedBitState(int iters, const char* regexp, const StringPiece& text) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->CanBitState()); + StringPiece sp[4]; // 4 because sp[0] is whole match. + for (int i = 0; i < iters; i++) + CHECK(prog->SearchBitState(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 4)); + delete prog; + re->Decref(); +} + +void Parse3CachedBacktrack(int iters, const char* regexp, const StringPiece& text) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + StringPiece sp[4]; // 4 because sp[0] is whole match. + for (int i = 0; i < iters; i++) + CHECK(prog->UnsafeSearchBacktrack(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 4)); + delete prog; + re->Decref(); +} + +void Parse3CachedPCRE(int iters, const char* regexp, const StringPiece& text) { + PCRE re(regexp, PCRE::UTF8); + CHECK_EQ(re.error(), ""); + StringPiece sp1, sp2, sp3; + for (int i = 0; i < iters; i++) { + CHECK(PCRE::FullMatch(text, re, &sp1, &sp2, &sp3)); + } +} + +void Parse3CachedRE2(int iters, const char* regexp, const StringPiece& text) { + RE2 re(regexp); + CHECK_EQ(re.error(), ""); + StringPiece sp1, sp2, sp3; + for (int i = 0; i < iters; i++) { + CHECK(RE2::FullMatch(text, re, &sp1, &sp2, &sp3)); + } +} + + +// Runs implementation to full match regexp against text, +// extracting three submatches. Expects match always. + +void Parse1NFA(int iters, const char* regexp, const StringPiece& text) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + StringPiece sp[2]; // 2 because sp[0] is whole match. + CHECK(prog->SearchNFA(text, StringPiece(), Prog::kAnchored, + Prog::kFullMatch, sp, 2)); + delete prog; + re->Decref(); + } +} + +void Parse1OnePass(int iters, const char* regexp, const StringPiece& text) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->IsOnePass()); + StringPiece sp[2]; // 2 because sp[0] is whole match. + CHECK(prog->SearchOnePass(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 2)); + delete prog; + re->Decref(); + } +} + +void Parse1BitState(int iters, const char* regexp, const StringPiece& text) { + for (int i = 0; i < iters; i++) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->CanBitState()); + StringPiece sp[2]; // 2 because sp[0] is whole match. + CHECK(prog->SearchBitState(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 2)); + delete prog; + re->Decref(); + } +} + +void Parse1PCRE(int iters, const char* regexp, const StringPiece& text) { + for (int i = 0; i < iters; i++) { + PCRE re(regexp, PCRE::UTF8); + CHECK_EQ(re.error(), ""); + StringPiece sp1; + CHECK(PCRE::FullMatch(text, re, &sp1)); + } +} + +void Parse1RE2(int iters, const char* regexp, const StringPiece& text) { + for (int i = 0; i < iters; i++) { + RE2 re(regexp); + CHECK_EQ(re.error(), ""); + StringPiece sp1; + CHECK(RE2::FullMatch(text, re, &sp1)); + } +} + +void Parse1CachedNFA(int iters, const char* regexp, const StringPiece& text) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + StringPiece sp[2]; // 2 because sp[0] is whole match. + for (int i = 0; i < iters; i++) { + CHECK(prog->SearchNFA(text, StringPiece(), Prog::kAnchored, + Prog::kFullMatch, sp, 2)); + } + delete prog; + re->Decref(); +} + +void Parse1CachedOnePass(int iters, const char* regexp, const StringPiece& text) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->IsOnePass()); + StringPiece sp[2]; // 2 because sp[0] is whole match. + for (int i = 0; i < iters; i++) + CHECK(prog->SearchOnePass(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 2)); + delete prog; + re->Decref(); +} + +void Parse1CachedBitState(int iters, const char* regexp, const StringPiece& text) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + CHECK(prog->CanBitState()); + StringPiece sp[2]; // 2 because sp[0] is whole match. + for (int i = 0; i < iters; i++) + CHECK(prog->SearchBitState(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 2)); + delete prog; + re->Decref(); +} + +void Parse1CachedBacktrack(int iters, const char* regexp, const StringPiece& text) { + Regexp* re = Regexp::Parse(regexp, Regexp::LikePerl, NULL); + CHECK(re); + Prog* prog = re->CompileToProg(0); + CHECK(prog); + StringPiece sp[2]; // 2 because sp[0] is whole match. + for (int i = 0; i < iters; i++) + CHECK(prog->UnsafeSearchBacktrack(text, text, Prog::kAnchored, Prog::kFullMatch, sp, 2)); + delete prog; + re->Decref(); +} + +void Parse1CachedPCRE(int iters, const char* regexp, const StringPiece& text) { + PCRE re(regexp, PCRE::UTF8); + CHECK_EQ(re.error(), ""); + StringPiece sp1; + for (int i = 0; i < iters; i++) { + CHECK(PCRE::FullMatch(text, re, &sp1)); + } +} + +void Parse1CachedRE2(int iters, const char* regexp, const StringPiece& text) { + RE2 re(regexp); + CHECK_EQ(re.error(), ""); + StringPiece sp1; + for (int i = 0; i < iters; i++) { + CHECK(RE2::FullMatch(text, re, &sp1)); + } +} + +void SearchParse2CachedPCRE(int iters, const char* regexp, + const StringPiece& text) { + PCRE re(regexp, PCRE::UTF8); + CHECK_EQ(re.error(), ""); + for (int i = 0; i < iters; i++) { + StringPiece sp1, sp2; + CHECK(PCRE::PartialMatch(text, re, &sp1, &sp2)); + } +} + +void SearchParse2CachedRE2(int iters, const char* regexp, + const StringPiece& text) { + RE2 re(regexp); + CHECK_EQ(re.error(), ""); + for (int i = 0; i < iters; i++) { + StringPiece sp1, sp2; + CHECK(RE2::PartialMatch(text, re, &sp1, &sp2)); + } +} + +void SearchParse1CachedPCRE(int iters, const char* regexp, + const StringPiece& text) { + PCRE re(regexp, PCRE::UTF8); + CHECK_EQ(re.error(), ""); + for (int i = 0; i < iters; i++) { + StringPiece sp1; + CHECK(PCRE::PartialMatch(text, re, &sp1)); + } +} + +void SearchParse1CachedRE2(int iters, const char* regexp, + const StringPiece& text) { + RE2 re(regexp); + CHECK_EQ(re.error(), ""); + for (int i = 0; i < iters; i++) { + StringPiece sp1; + CHECK(RE2::PartialMatch(text, re, &sp1)); + } +} + +void EmptyPartialMatchPCRE(int n) { + PCRE re(""); + for (int i = 0; i < n; i++) { + PCRE::PartialMatch("", re); + } +} + +void EmptyPartialMatchRE2(int n) { + RE2 re(""); + for (int i = 0; i < n; i++) { + RE2::PartialMatch("", re); + } +} +#ifdef USEPCRE +BENCHMARK(EmptyPartialMatchPCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(EmptyPartialMatchRE2)->ThreadRange(1, NumCPUs()); + +void SimplePartialMatchPCRE(int n) { + PCRE re("abcdefg"); + for (int i = 0; i < n; i++) { + PCRE::PartialMatch("abcdefg", re); + } +} + +void SimplePartialMatchRE2(int n) { + RE2 re("abcdefg"); + for (int i = 0; i < n; i++) { + RE2::PartialMatch("abcdefg", re); + } +} +#ifdef USEPCRE +BENCHMARK(SimplePartialMatchPCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(SimplePartialMatchRE2)->ThreadRange(1, NumCPUs()); + +static std::string http_text = + "GET /asdfhjasdhfasdlfhasdflkjasdfkljasdhflaskdjhf" + "alksdjfhasdlkfhasdlkjfhasdljkfhadsjklf HTTP/1.1"; + +void HTTPPartialMatchPCRE(int n) { + StringPiece a; + PCRE re("(?-s)^(?:GET|POST) +([^ ]+) HTTP"); + for (int i = 0; i < n; i++) { + PCRE::PartialMatch(http_text, re, &a); + } +} + +void HTTPPartialMatchRE2(int n) { + StringPiece a; + RE2 re("(?-s)^(?:GET|POST) +([^ ]+) HTTP"); + for (int i = 0; i < n; i++) { + RE2::PartialMatch(http_text, re, &a); + } +} + +#ifdef USEPCRE +BENCHMARK(HTTPPartialMatchPCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(HTTPPartialMatchRE2)->ThreadRange(1, NumCPUs()); + +static std::string smallhttp_text = + "GET /abc HTTP/1.1"; + +void SmallHTTPPartialMatchPCRE(int n) { + StringPiece a; + PCRE re("(?-s)^(?:GET|POST) +([^ ]+) HTTP"); + for (int i = 0; i < n; i++) { + PCRE::PartialMatch(smallhttp_text, re, &a); + } +} + +void SmallHTTPPartialMatchRE2(int n) { + StringPiece a; + RE2 re("(?-s)^(?:GET|POST) +([^ ]+) HTTP"); + for (int i = 0; i < n; i++) { + RE2::PartialMatch(smallhttp_text, re, &a); + } +} + +#ifdef USEPCRE +BENCHMARK(SmallHTTPPartialMatchPCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(SmallHTTPPartialMatchRE2)->ThreadRange(1, NumCPUs()); + +void DotMatchPCRE(int n) { + StringPiece a; + PCRE re("(?-s)^(.+)"); + for (int i = 0; i < n; i++) { + PCRE::PartialMatch(http_text, re, &a); + } +} + +void DotMatchRE2(int n) { + StringPiece a; + RE2 re("(?-s)^(.+)"); + for (int i = 0; i < n; i++) { + RE2::PartialMatch(http_text, re, &a); + } +} + +#ifdef USEPCRE +BENCHMARK(DotMatchPCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(DotMatchRE2)->ThreadRange(1, NumCPUs()); + +void ASCIIMatchPCRE(int n) { + StringPiece a; + PCRE re("(?-s)^([ -~]+)"); + for (int i = 0; i < n; i++) { + PCRE::PartialMatch(http_text, re, &a); + } +} + +void ASCIIMatchRE2(int n) { + StringPiece a; + RE2 re("(?-s)^([ -~]+)"); + for (int i = 0; i < n; i++) { + RE2::PartialMatch(http_text, re, &a); + } +} + +#ifdef USEPCRE +BENCHMARK(ASCIIMatchPCRE)->ThreadRange(1, NumCPUs()); +#endif +BENCHMARK(ASCIIMatchRE2)->ThreadRange(1, NumCPUs()); + +void FullMatchPCRE(int iter, int n, const char *regexp) { + StopBenchmarkTiming(); + std::string s; + MakeText(&s, n); + s += "ABCDEFGHIJ"; + BenchmarkMemoryUsage(); + PCRE re(regexp); + StartBenchmarkTiming(); + for (int i = 0; i < iter; i++) + CHECK(PCRE::FullMatch(s, re)); + SetBenchmarkBytesProcessed(static_cast(iter)*n); +} + +void FullMatchRE2(int iter, int n, const char *regexp) { + StopBenchmarkTiming(); + std::string s; + MakeText(&s, n); + s += "ABCDEFGHIJ"; + BenchmarkMemoryUsage(); + RE2 re(regexp, RE2::Latin1); + StartBenchmarkTiming(); + for (int i = 0; i < iter; i++) + CHECK(RE2::FullMatch(s, re)); + SetBenchmarkBytesProcessed(static_cast(iter)*n); +} + +void FullMatch_DotStar_CachedPCRE(int i, int n) { FullMatchPCRE(i, n, "(?s).*"); } +void FullMatch_DotStar_CachedRE2(int i, int n) { FullMatchRE2(i, n, "(?s).*"); } + +void FullMatch_DotStarDollar_CachedPCRE(int i, int n) { FullMatchPCRE(i, n, "(?s).*$"); } +void FullMatch_DotStarDollar_CachedRE2(int i, int n) { FullMatchRE2(i, n, "(?s).*$"); } + +void FullMatch_DotStarCapture_CachedPCRE(int i, int n) { FullMatchPCRE(i, n, "(?s)((.*)()()($))"); } +void FullMatch_DotStarCapture_CachedRE2(int i, int n) { FullMatchRE2(i, n, "(?s)((.*)()()($))"); } + +#ifdef USEPCRE +BENCHMARK_RANGE(FullMatch_DotStar_CachedPCRE, 8, 2<<20); +#endif +BENCHMARK_RANGE(FullMatch_DotStar_CachedRE2, 8, 2<<20); + +#ifdef USEPCRE +BENCHMARK_RANGE(FullMatch_DotStarDollar_CachedPCRE, 8, 2<<20); +#endif +BENCHMARK_RANGE(FullMatch_DotStarDollar_CachedRE2, 8, 2<<20); + +#ifdef USEPCRE +BENCHMARK_RANGE(FullMatch_DotStarCapture_CachedPCRE, 8, 2<<20); +#endif +BENCHMARK_RANGE(FullMatch_DotStarCapture_CachedRE2, 8, 2<<20); + +void PossibleMatchRangeCommon(int iter, const char* regexp) { + StopBenchmarkTiming(); + RE2 re(regexp); + StartBenchmarkTiming(); + std::string min; + std::string max; + const int kMaxLen = 16; + for (int i = 0; i < iter; i++) { + CHECK(re.PossibleMatchRange(&min, &max, kMaxLen)); + } +} + +void PossibleMatchRange_Trivial(int i) { + PossibleMatchRangeCommon(i, ".*"); +} +void PossibleMatchRange_Complex(int i) { + PossibleMatchRangeCommon(i, "^abc[def]?[gh]{1,2}.*"); +} +void PossibleMatchRange_Prefix(int i) { + PossibleMatchRangeCommon(i, "^some_random_prefix.*"); +} +void PossibleMatchRange_NoProg(int i) { + PossibleMatchRangeCommon(i, "^some_random_string$"); +} + +BENCHMARK(PossibleMatchRange_Trivial); +BENCHMARK(PossibleMatchRange_Complex); +BENCHMARK(PossibleMatchRange_Prefix); +BENCHMARK(PossibleMatchRange_NoProg); + +} // namespace re2 diff --git a/extern/re2/re2/testing/regexp_generator.cc b/extern/re2/re2/testing/regexp_generator.cc new file mode 100644 index 0000000000..1e4d3da990 --- /dev/null +++ b/extern/re2/re2/testing/regexp_generator.cc @@ -0,0 +1,276 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Regular expression generator: generates all possible +// regular expressions within parameters (see regexp_generator.h for details). + +// The regexp generator first generates a sequence of commands in a simple +// postfix language. Each command in the language is a string, +// like "a" or "%s*" or "%s|%s". +// +// To evaluate a command, enough arguments are popped from the value stack to +// plug into the %s slots. Then the result is pushed onto the stack. +// For example, the command sequence +// a b %s%s c +// results in the stack +// ab c +// +// GeneratePostfix generates all possible command sequences. +// Then RunPostfix turns each sequence into a regular expression +// and passes the regexp to HandleRegexp. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util/test.h" +#include "util/logging.h" +#include "util/strutil.h" +#include "util/utf.h" +#include "re2/testing/regexp_generator.h" + +namespace re2 { + +// Returns a vector of the egrep regexp operators. +const std::vector& RegexpGenerator::EgrepOps() { + static const char *ops[] = { + "%s%s", + "%s|%s", + "%s*", + "%s+", + "%s?", + "%s\\C*", + }; + static std::vector v(ops, ops + arraysize(ops)); + return v; +} + +RegexpGenerator::RegexpGenerator(int maxatoms, int maxops, + const std::vector& atoms, + const std::vector& ops) + : maxatoms_(maxatoms), maxops_(maxops), atoms_(atoms), ops_(ops) { + // Degenerate case. + if (atoms_.empty()) + maxatoms_ = 0; + if (ops_.empty()) + maxops_ = 0; +} + +// Generates all possible regular expressions (within the parameters), +// calling HandleRegexp for each one. +void RegexpGenerator::Generate() { + std::vector postfix; + GeneratePostfix(&postfix, 0, 0, 0); +} + +// Generates random regular expressions, calling HandleRegexp for each one. +void RegexpGenerator::GenerateRandom(int32_t seed, int n) { + rng_.seed(seed); + + for (int i = 0; i < n; i++) { + std::vector postfix; + GenerateRandomPostfix(&postfix, 0, 0, 0); + } +} + +// Counts and returns the number of occurrences of "%s" in s. +static int CountArgs(const std::string& s) { + const char *p = s.c_str(); + int n = 0; + while ((p = strstr(p, "%s")) != NULL) { + p += 2; + n++; + } + return n; +} + +// Generates all possible postfix command sequences. +// Each sequence is handed off to RunPostfix to generate a regular expression. +// The arguments are: +// post: the current postfix sequence +// nstk: the number of elements that would be on the stack after executing +// the sequence +// ops: the number of operators used in the sequence +// atoms: the number of atoms used in the sequence +// For example, if post were ["a", "b", "%s%s", "c"], +// then nstk = 2, ops = 1, atoms = 3. +// +// The initial call should be GeneratePostfix([empty vector], 0, 0, 0). +// +void RegexpGenerator::GeneratePostfix(std::vector* post, + int nstk, int ops, int atoms) { + if (nstk == 1) + RunPostfix(*post); + + // Early out: if used too many operators or can't + // get back down to a single expression on the stack + // using binary operators, give up. + if (ops + nstk - 1 > maxops_) + return; + + // Add atoms if there is room. + if (atoms < maxatoms_) { + for (size_t i = 0; i < atoms_.size(); i++) { + post->push_back(atoms_[i]); + GeneratePostfix(post, nstk + 1, ops, atoms + 1); + post->pop_back(); + } + } + + // Add operators if there are enough arguments. + if (ops < maxops_) { + for (size_t i = 0; i < ops_.size(); i++) { + const std::string& fmt = ops_[i]; + int nargs = CountArgs(fmt); + if (nargs <= nstk) { + post->push_back(fmt); + GeneratePostfix(post, nstk - nargs + 1, ops + 1, atoms); + post->pop_back(); + } + } + } +} + +// Generates a random postfix command sequence. +// Stops and returns true once a single sequence has been generated. +bool RegexpGenerator::GenerateRandomPostfix(std::vector* post, + int nstk, int ops, int atoms) { + std::uniform_int_distribution random_stop(0, maxatoms_ - atoms); + std::uniform_int_distribution random_bit(0, 1); + std::uniform_int_distribution random_ops_index( + 0, static_cast(ops_.size()) - 1); + std::uniform_int_distribution random_atoms_index( + 0, static_cast(atoms_.size()) - 1); + + for (;;) { + // Stop if we get to a single element, but only sometimes. + if (nstk == 1 && random_stop(rng_) == 0) { + RunPostfix(*post); + return true; + } + + // Early out: if used too many operators or can't + // get back down to a single expression on the stack + // using binary operators, give up. + if (ops + nstk - 1 > maxops_) + return false; + + // Add operators if there are enough arguments. + if (ops < maxops_ && random_bit(rng_) == 0) { + const std::string& fmt = ops_[random_ops_index(rng_)]; + int nargs = CountArgs(fmt); + if (nargs <= nstk) { + post->push_back(fmt); + bool ret = GenerateRandomPostfix(post, nstk - nargs + 1, + ops + 1, atoms); + post->pop_back(); + if (ret) + return true; + } + } + + // Add atoms if there is room. + if (atoms < maxatoms_ && random_bit(rng_) == 0) { + post->push_back(atoms_[random_atoms_index(rng_)]); + bool ret = GenerateRandomPostfix(post, nstk + 1, ops, atoms + 1); + post->pop_back(); + if (ret) + return true; + } + } +} + +// Interprets the postfix command sequence to create a regular expression +// passed to HandleRegexp. The results of operators like %s|%s are wrapped +// in (?: ) to avoid needing to maintain a precedence table. +void RegexpGenerator::RunPostfix(const std::vector& post) { + std::stack regexps; + for (size_t i = 0; i < post.size(); i++) { + switch (CountArgs(post[i])) { + default: + LOG(FATAL) << "Bad operator: " << post[i]; + case 0: + regexps.push(post[i]); + break; + case 1: { + std::string a = regexps.top(); + regexps.pop(); + regexps.push("(?:" + StringPrintf(post[i].c_str(), a.c_str()) + ")"); + break; + } + case 2: { + std::string b = regexps.top(); + regexps.pop(); + std::string a = regexps.top(); + regexps.pop(); + regexps.push("(?:" + + StringPrintf(post[i].c_str(), a.c_str(), b.c_str()) + + ")"); + break; + } + } + } + + if (regexps.size() != 1) { + // Internal error - should never happen. + printf("Bad regexp program:\n"); + for (size_t i = 0; i < post.size(); i++) { + printf(" %s\n", CEscape(post[i]).c_str()); + } + printf("Stack after running program:\n"); + while (!regexps.empty()) { + printf(" %s\n", CEscape(regexps.top()).c_str()); + regexps.pop(); + } + LOG(FATAL) << "Bad regexp program."; + } + + HandleRegexp(regexps.top()); + HandleRegexp("^(?:" + regexps.top() + ")$"); + HandleRegexp("^(?:" + regexps.top() + ")"); + HandleRegexp("(?:" + regexps.top() + ")$"); +} + +// Split s into an vector of strings, one for each UTF-8 character. +std::vector Explode(const StringPiece& s) { + std::vector v; + + for (const char *q = s.begin(); q < s.end(); ) { + const char* p = q; + Rune r; + q += chartorune(&r, q); + v.push_back(std::string(p, q - p)); + } + + return v; +} + +// Split string everywhere a substring is found, returning +// vector of pieces. +std::vector Split(const StringPiece& sep, const StringPiece& s) { + std::vector v; + + if (sep.size() == 0) + return Explode(s); + + const char *p = s.begin(); + for (const char *q = s.begin(); q + sep.size() <= s.end(); q++) { + if (StringPiece(q, sep.size()) == sep) { + v.push_back(std::string(p, q - p)); + p = q + sep.size(); + q = p - 1; // -1 for ++ in loop + continue; + } + } + if (p < s.end()) + v.push_back(std::string(p, s.end() - p)); + return v; +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/regexp_generator.h b/extern/re2/re2/testing/regexp_generator.h new file mode 100644 index 0000000000..7d72aff889 --- /dev/null +++ b/extern/re2/re2/testing/regexp_generator.h @@ -0,0 +1,77 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_TESTING_REGEXP_GENERATOR_H_ +#define RE2_TESTING_REGEXP_GENERATOR_H_ + +// Regular expression generator: generates all possible +// regular expressions within given parameters (see below for details). + +#include +#include +#include +#include + +#include "util/util.h" +#include "re2/stringpiece.h" + +namespace re2 { + +// Regular expression generator. +// +// Given a set of atom expressions like "a", "b", or "." +// and operators like "%s*", generates all possible regular expressions +// using at most maxbases base expressions and maxops operators. +// For each such expression re, calls HandleRegexp(re). +// +// Callers are expected to subclass RegexpGenerator and provide HandleRegexp. +// +class RegexpGenerator { + public: + RegexpGenerator(int maxatoms, int maxops, + const std::vector& atoms, + const std::vector& ops); + virtual ~RegexpGenerator() {} + + // Generates all the regular expressions, calling HandleRegexp(re) for each. + void Generate(); + + // Generates n random regular expressions, calling HandleRegexp(re) for each. + void GenerateRandom(int32_t seed, int n); + + // Handles a regular expression. Must be provided by subclass. + virtual void HandleRegexp(const std::string& regexp) = 0; + + // The egrep regexp operators: * + ? | and concatenation. + static const std::vector& EgrepOps(); + + private: + void RunPostfix(const std::vector& post); + void GeneratePostfix(std::vector* post, + int nstk, int ops, int lits); + bool GenerateRandomPostfix(std::vector* post, + int nstk, int ops, int lits); + + int maxatoms_; // Maximum number of atoms allowed in expr. + int maxops_; // Maximum number of ops allowed in expr. + std::vector atoms_; // Possible atoms. + std::vector ops_; // Possible ops. + std::minstd_rand0 rng_; // Random number generator. + + RegexpGenerator(const RegexpGenerator&) = delete; + RegexpGenerator& operator=(const RegexpGenerator&) = delete; +}; + +// Helpers for preparing arguments to RegexpGenerator constructor. + +// Returns one string for each character in s. +std::vector Explode(const StringPiece& s); + +// Splits string everywhere sep is found, returning +// vector of pieces. +std::vector Split(const StringPiece& sep, const StringPiece& s); + +} // namespace re2 + +#endif // RE2_TESTING_REGEXP_GENERATOR_H_ diff --git a/extern/re2/re2/testing/regexp_test.cc b/extern/re2/re2/testing/regexp_test.cc new file mode 100644 index 0000000000..f7e7e922e6 --- /dev/null +++ b/extern/re2/re2/testing/regexp_test.cc @@ -0,0 +1,86 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test parse.cc, dump.cc, and tostring.cc. + +#include +#include +#include +#include + +#include "util/test.h" +#include "util/logging.h" +#include "re2/regexp.h" + +namespace re2 { + +// Test that overflowed ref counts work. +TEST(Regexp, BigRef) { + Regexp* re; + re = Regexp::Parse("x", Regexp::NoParseFlags, NULL); + for (int i = 0; i < 100000; i++) + re->Incref(); + for (int i = 0; i < 100000; i++) + re->Decref(); + ASSERT_EQ(re->Ref(), 1); + re->Decref(); +} + +// Test that very large Concats work. +// Depends on overflowed ref counts working. +TEST(Regexp, BigConcat) { + Regexp* x; + x = Regexp::Parse("x", Regexp::NoParseFlags, NULL); + std::vector v(90000, x); // ToString bails out at 100000 + for (size_t i = 0; i < v.size(); i++) + x->Incref(); + ASSERT_EQ(x->Ref(), 1 + static_cast(v.size())) << x->Ref(); + Regexp* re = Regexp::Concat(v.data(), static_cast(v.size()), + Regexp::NoParseFlags); + ASSERT_EQ(re->ToString(), std::string(v.size(), 'x')); + re->Decref(); + ASSERT_EQ(x->Ref(), 1) << x->Ref(); + x->Decref(); +} + +TEST(Regexp, NamedCaptures) { + Regexp* x; + RegexpStatus status; + x = Regexp::Parse( + "(?Pa+)|(e)(?Pw*)+(?Pb+)", Regexp::PerlX, &status); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(4, x->NumCaptures()); + const std::map* have = x->NamedCaptures(); + EXPECT_TRUE(have != NULL); + EXPECT_EQ(2, have->size()); // there are only two named groups in + // the regexp: 'g1' and 'g2'. + std::map want; + want["g1"] = 1; + want["g2"] = 3; + EXPECT_EQ(want, *have); + x->Decref(); + delete have; +} + +TEST(Regexp, CaptureNames) { + Regexp* x; + RegexpStatus status; + x = Regexp::Parse( + "(?Pa+)|(e)(?Pw*)+(?Pb+)", Regexp::PerlX, &status); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(4, x->NumCaptures()); + const std::map* have = x->CaptureNames(); + EXPECT_TRUE(have != NULL); + EXPECT_EQ(3, have->size()); + std::map want; + want[1] = "g1"; + want[3] = "g2"; + want[4] = "g1"; + + EXPECT_EQ(want, *have); + x->Decref(); + delete have; +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/required_prefix_test.cc b/extern/re2/re2/testing/required_prefix_test.cc new file mode 100644 index 0000000000..54600456a8 --- /dev/null +++ b/extern/re2/re2/testing/required_prefix_test.cc @@ -0,0 +1,72 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include + +#include "util/test.h" +#include "util/logging.h" +#include "re2/regexp.h" + +namespace re2 { + +struct PrefixTest { + const char* regexp; + bool return_value; + const char* prefix; + bool foldcase; + const char* suffix; +}; + +static PrefixTest tests[] = { + // If the regexp is missing a ^, there's no required prefix. + { "abc", false }, + { "", false }, + { "(?m)^", false }, + + // If the regexp immediately goes into + // something not a literal match, there's no required prefix. + { "^(abc)", false }, + { "^a*", false }, + + // Otherwise, it should work. + { "^abc$", true, "abc", false, "(?-m:$)" }, + { "^abc", true, "abc", false, "" }, + { "^(?i)abc", true, "abc", true, "" }, + { "^abcd*", true, "abc", false, "d*" }, + { "^[Aa][Bb]cd*", true, "ab", true, "cd*" }, + { "^ab[Cc]d*", true, "ab", false, "[Cc]d*" }, + { "^☺abc", true, "☺abc", false, "" }, +}; + +TEST(RequiredPrefix, SimpleTests) { + for (size_t i = 0; i < arraysize(tests); i++) { + const PrefixTest& t = tests[i]; + for (size_t j = 0; j < 2; j++) { + Regexp::ParseFlags flags = Regexp::LikePerl; + if (j == 0) + flags = flags | Regexp::Latin1; + Regexp* re = Regexp::Parse(t.regexp, flags, NULL); + ASSERT_TRUE(re != NULL) << " " << t.regexp; + + std::string p; + bool f; + Regexp* s; + ASSERT_EQ(t.return_value, re->RequiredPrefix(&p, &f, &s)) + << " " << t.regexp << " " << (j == 0 ? "latin1" : "utf") + << " " << re->Dump(); + if (t.return_value) { + ASSERT_EQ(p, std::string(t.prefix)) + << " " << t.regexp << " " << (j == 0 ? "latin1" : "utf"); + ASSERT_EQ(f, t.foldcase) + << " " << t.regexp << " " << (j == 0 ? "latin1" : "utf"); + ASSERT_EQ(s->ToString(), std::string(t.suffix)) + << " " << t.regexp << " " << (j == 0 ? "latin1" : "utf"); + s->Decref(); + } + re->Decref(); + } + } +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/search_test.cc b/extern/re2/re2/testing/search_test.cc new file mode 100644 index 0000000000..c20f501dae --- /dev/null +++ b/extern/re2/re2/testing/search_test.cc @@ -0,0 +1,332 @@ +// Copyright 2006-2007 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "util/test.h" +#include "re2/prog.h" +#include "re2/regexp.h" +#include "re2/testing/tester.h" +#include "re2/testing/exhaustive_tester.h" + +// For target `log' in the Makefile. +#ifndef LOGGING +#define LOGGING 0 +#endif + +namespace re2 { + +struct RegexpTest { + const char* regexp; + const char* text; +}; + +RegexpTest simple_tests[] = { + { "a", "a" }, + { "a", "zyzzyva" }, + { "a+", "aa" }, + { "(a+|b)+", "ab" }, + { "ab|cd", "xabcdx" }, + { "h.*od?", "hello\ngoodbye\n" }, + { "h.*o", "hello\ngoodbye\n" }, + { "h.*o", "goodbye\nhello\n" }, + { "h.*o", "hello world" }, + { "h.*o", "othello, world" }, + { "[^\\s\\S]", "aaaaaaa" }, + { "a", "aaaaaaa" }, + { "a*", "aaaaaaa" }, + { "a*", "" }, + { "ab|cd", "xabcdx" }, + { "a", "cab" }, + { "a*b", "cab" }, + { "((((((((((((((((((((x))))))))))))))))))))", "x" }, + { "[abcd]", "xxxabcdxxx" }, + { "[^x]", "xxxabcdxxx" }, + { "[abcd]+", "xxxabcdxxx" }, + { "[^x]+", "xxxabcdxxx" }, + { "(fo|foo)", "fo" }, + { "(foo|fo)", "foo" }, + + { "aa", "aA" }, + { "a", "Aa" }, + { "a", "A" }, + { "ABC", "abc" }, + { "abc", "XABCY" }, + { "ABC", "xabcy" }, + + // Make sure ^ and $ work. + // The pathological cases didn't work + // in the original grep code. + { "foo|bar|[A-Z]", "foo" }, + { "^(foo|bar|[A-Z])", "foo" }, + { "(foo|bar|[A-Z])$", "foo\n" }, + { "(foo|bar|[A-Z])$", "foo" }, + { "^(foo|bar|[A-Z])$", "foo\n" }, + { "^(foo|bar|[A-Z])$", "foo" }, + { "^(foo|bar|[A-Z])$", "bar" }, + { "^(foo|bar|[A-Z])$", "X" }, + { "^(foo|bar|[A-Z])$", "XY" }, + { "^(fo|foo)$", "fo" }, + { "^(fo|foo)$", "foo" }, + { "^^(fo|foo)$", "fo" }, + { "^^(fo|foo)$", "foo" }, + { "^$", "" }, + { "^$", "x" }, + { "^^$", "" }, + { "^$$", "" }, + { "^^$", "x" }, + { "^$$", "x" }, + { "^^$$", "" }, + { "^^$$", "x" }, + { "^^^^^^^^$$$$$$$$", "" }, + { "^", "x" }, + { "$", "x" }, + + // Word boundaries. + { "\\bfoo\\b", "nofoo foo that" }, + { "a\\b", "faoa x" }, + { "\\bbar", "bar x" }, + { "\\bbar", "foo\nbar x" }, + { "bar\\b", "foobar" }, + { "bar\\b", "foobar\nxxx" }, + { "(foo|bar|[A-Z])\\b", "foo" }, + { "(foo|bar|[A-Z])\\b", "foo\n" }, + { "\\b", "" }, + { "\\b", "x" }, + { "\\b(foo|bar|[A-Z])", "foo" }, + { "\\b(foo|bar|[A-Z])\\b", "X" }, + { "\\b(foo|bar|[A-Z])\\b", "XY" }, + { "\\b(foo|bar|[A-Z])\\b", "bar" }, + { "\\b(foo|bar|[A-Z])\\b", "foo" }, + { "\\b(foo|bar|[A-Z])\\b", "foo\n" }, + { "\\b(foo|bar|[A-Z])\\b", "ffoo bbar N x" }, + { "\\b(fo|foo)\\b", "fo" }, + { "\\b(fo|foo)\\b", "foo" }, + { "\\b\\b", "" }, + { "\\b\\b", "x" }, + { "\\b$", "" }, + { "\\b$", "x" }, + { "\\b$", "y x" }, + { "\\b.$", "x" }, + { "^\\b(fo|foo)\\b", "fo" }, + { "^\\b(fo|foo)\\b", "foo" }, + { "^\\b", "" }, + { "^\\b", "x" }, + { "^\\b\\b", "" }, + { "^\\b\\b", "x" }, + { "^\\b$", "" }, + { "^\\b$", "x" }, + { "^\\b.$", "x" }, + { "^\\b.\\b$", "x" }, + { "^^^^^^^^\\b$$$$$$$", "" }, + { "^^^^^^^^\\b.$$$$$$", "x" }, + { "^^^^^^^^\\b$$$$$$$", "x" }, + + // Non-word boundaries. + { "\\Bfoo\\B", "n foo xfoox that" }, + { "a\\B", "faoa x" }, + { "\\Bbar", "bar x" }, + { "\\Bbar", "foo\nbar x" }, + { "bar\\B", "foobar" }, + { "bar\\B", "foobar\nxxx" }, + { "(foo|bar|[A-Z])\\B", "foox" }, + { "(foo|bar|[A-Z])\\B", "foo\n" }, + { "\\B", "" }, + { "\\B", "x" }, + { "\\B(foo|bar|[A-Z])", "foo" }, + { "\\B(foo|bar|[A-Z])\\B", "xXy" }, + { "\\B(foo|bar|[A-Z])\\B", "XY" }, + { "\\B(foo|bar|[A-Z])\\B", "XYZ" }, + { "\\B(foo|bar|[A-Z])\\B", "abara" }, + { "\\B(foo|bar|[A-Z])\\B", "xfoo_" }, + { "\\B(foo|bar|[A-Z])\\B", "xfoo\n" }, + { "\\B(foo|bar|[A-Z])\\B", "foo bar vNx" }, + { "\\B(fo|foo)\\B", "xfoo" }, + { "\\B(foo|fo)\\B", "xfooo" }, + { "\\B\\B", "" }, + { "\\B\\B", "x" }, + { "\\B$", "" }, + { "\\B$", "x" }, + { "\\B$", "y x" }, + { "\\B.$", "x" }, + { "^\\B(fo|foo)\\B", "fo" }, + { "^\\B(fo|foo)\\B", "foo" }, + { "^\\B", "" }, + { "^\\B", "x" }, + { "^\\B\\B", "" }, + { "^\\B\\B", "x" }, + { "^\\B$", "" }, + { "^\\B$", "x" }, + { "^\\B.$", "x" }, + { "^\\B.\\B$", "x" }, + { "^^^^^^^^\\B$$$$$$$", "" }, + { "^^^^^^^^\\B.$$$$$$", "x" }, + { "^^^^^^^^\\B$$$$$$$", "x" }, + + // PCRE uses only ASCII for \b computation. + // All non-ASCII are *not* word characters. + { "\\bx\\b", "x" }, + { "\\bx\\b", "x>" }, + { "\\bx\\b", "" }, + { "\\bx\\b", "ax" }, + { "\\bx\\b", "xb" }, + { "\\bx\\b", "axb" }, + { "\\bx\\b", "«x" }, + { "\\bx\\b", "x»" }, + { "\\bx\\b", "«x»" }, + { "\\bx\\b", "axb" }, + { "\\bx\\b", "áxβ" }, + { "\\Bx\\B", "axb" }, + { "\\Bx\\B", "áxβ" }, + + // Weird boundary cases. + { "^$^$", "" }, + { "^$^", "" }, + { "$^$", "" }, + + { "^$^$", "x" }, + { "^$^", "x" }, + { "$^$", "x" }, + + { "^$^$", "x\ny" }, + { "^$^", "x\ny" }, + { "$^$", "x\ny" }, + + { "^$^$", "x\n\ny" }, + { "^$^", "x\n\ny" }, + { "$^$", "x\n\ny" }, + + { "^(foo\\$)$", "foo$bar" }, + { "(foo\\$)", "foo$bar" }, + { "^...$", "abc" }, + + // UTF-8 + { "^\xe6\x9c\xac$", "\xe6\x9c\xac" }, + { "^...$", "\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e" }, + { "^...$", ".\xe6\x9c\xac." }, + + { "^\\C\\C\\C$", "\xe6\x9c\xac" }, + { "^\\C$", "\xe6\x9c\xac" }, + { "^\\C\\C\\C$", "\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e" }, + + // Latin1 + { "^...$", "\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e" }, + { "^.........$", "\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e" }, + { "^...$", ".\xe6\x9c\xac." }, + { "^.....$", ".\xe6\x9c\xac." }, + + // Perl v Posix + { "\\B(fo|foo)\\B", "xfooo" }, + { "(fo|foo)", "foo" }, + + // Octal escapes. + { "\\141", "a" }, + { "\\060", "0" }, + { "\\0600", "00" }, + { "\\608", "08" }, + { "\\01", "\01" }, + { "\\018", "\01" "8" }, + + // Hexadecimal escapes + { "\\x{61}", "a" }, + { "\\x61", "a" }, + { "\\x{00000061}", "a" }, + + // Unicode scripts. + { "\\p{Greek}+", "aαβb" }, + { "\\P{Greek}+", "aαβb" }, + { "\\p{^Greek}+", "aαβb" }, + { "\\P{^Greek}+", "aαβb" }, + + // Unicode properties. Nd is decimal number. N is any number. + { "[^0-9]+", "abc123" }, + { "\\p{Nd}+", "abc123²³¼½¾₀₉" }, + { "\\p{^Nd}+", "abc123²³¼½¾₀₉" }, + { "\\P{Nd}+", "abc123²³¼½¾₀₉" }, + { "\\P{^Nd}+", "abc123²³¼½¾₀₉" }, + { "\\pN+", "abc123²³¼½¾₀₉" }, + { "\\p{N}+", "abc123²³¼½¾₀₉" }, + { "\\p{^N}+", "abc123²³¼½¾₀₉" }, + + { "\\p{Any}+", "abc123" }, + + // Character classes & case folding. + { "(?i)[@-A]+", "@AaB" }, // matches @Aa but not B + { "(?i)[A-Z]+", "aAzZ" }, + { "(?i)[^\\\\]+", "Aa\\" }, // \\ is between A-Z and a-z - + // splits the ranges in an interesting way. + + // would like to use, but PCRE mishandles in full-match, non-greedy mode + // { "(?i)[\\\\]+", "Aa" }, + + { "(?i)[acegikmoqsuwy]+", "acegikmoqsuwyACEGIKMOQSUWY" }, + + // Character classes & case folding. + { "[@-A]+", "@AaB" }, + { "[A-Z]+", "aAzZ" }, + { "[^\\\\]+", "Aa\\" }, + { "[acegikmoqsuwy]+", "acegikmoqsuwyACEGIKMOQSUWY" }, + + // Anchoring. (^abc in aabcdef was a former bug) + // The tester checks for a match in the text and + // subpieces of the text with a byte removed on either side. + { "^abc", "abcdef" }, + { "^abc", "aabcdef" }, + { "^[ay]*[bx]+c", "abcdef" }, + { "^[ay]*[bx]+c", "aabcdef" }, + { "def$", "abcdef" }, + { "def$", "abcdeff" }, + { "d[ex][fy]$", "abcdef" }, + { "d[ex][fy]$", "abcdeff" }, + { "[dz][ex][fy]$", "abcdef" }, + { "[dz][ex][fy]$", "abcdeff" }, + { "(?m)^abc", "abcdef" }, + { "(?m)^abc", "aabcdef" }, + { "(?m)^[ay]*[bx]+c", "abcdef" }, + { "(?m)^[ay]*[bx]+c", "aabcdef" }, + { "(?m)def$", "abcdef" }, + { "(?m)def$", "abcdeff" }, + { "(?m)d[ex][fy]$", "abcdef" }, + { "(?m)d[ex][fy]$", "abcdeff" }, + { "(?m)[dz][ex][fy]$", "abcdef" }, + { "(?m)[dz][ex][fy]$", "abcdeff" }, + { "^", "a" }, + { "^^", "a" }, + + // Context. + // The tester checks for a match in the text and + // subpieces of the text with a byte removed on either side. + { "a", "a" }, + { "ab*", "a" }, + { "a\\C*", "a" }, + { "a\\C+", "a" }, + { "a\\C?", "a" }, + { "a\\C*?", "a" }, + { "a\\C+?", "a" }, + { "a\\C??", "a" }, + + // Former bugs. + { "a\\C*|ba\\C", "baba" }, + { "\\w*I\\w*", "Inc." }, +}; + +TEST(Regexp, SearchTests) { + int failures = 0; + for (size_t i = 0; i < arraysize(simple_tests); i++) { + const RegexpTest& t = simple_tests[i]; + if (!TestRegexpOnText(t.regexp, t.text)) + failures++; + + if (LOGGING) { + // Build a dummy ExhaustiveTest call that will trigger just + // this one test, so that we log the test case. + std::vector atom, alpha, ops; + atom.push_back(t.regexp); + alpha.push_back(t.text); + ExhaustiveTest(1, 0, atom, ops, 1, alpha, "", ""); + } + } + EXPECT_EQ(failures, 0); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/set_test.cc b/extern/re2/re2/testing/set_test.cc new file mode 100644 index 0000000000..61d1cf295f --- /dev/null +++ b/extern/re2/re2/testing/set_test.cc @@ -0,0 +1,204 @@ +// Copyright 2010 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include +#include + +#include "util/test.h" +#include "util/logging.h" +#include "re2/re2.h" +#include "re2/set.h" + +namespace re2 { + +TEST(Set, Unanchored) { + RE2::Set s(RE2::DefaultOptions, RE2::UNANCHORED); + + ASSERT_EQ(s.Add("foo", NULL), 0); + ASSERT_EQ(s.Add("(", NULL), -1); + ASSERT_EQ(s.Add("bar", NULL), 1); + ASSERT_EQ(s.Compile(), true); + + ASSERT_EQ(s.Match("foobar", NULL), true); + ASSERT_EQ(s.Match("fooba", NULL), true); + ASSERT_EQ(s.Match("oobar", NULL), true); + + std::vector v; + ASSERT_EQ(s.Match("foobar", &v), true); + ASSERT_EQ(v.size(), 2); + ASSERT_EQ(v[0], 0); + ASSERT_EQ(v[1], 1); + + ASSERT_EQ(s.Match("fooba", &v), true); + ASSERT_EQ(v.size(), 1); + ASSERT_EQ(v[0], 0); + + ASSERT_EQ(s.Match("oobar", &v), true); + ASSERT_EQ(v.size(), 1); + ASSERT_EQ(v[0], 1); +} + +TEST(Set, UnanchoredFactored) { + RE2::Set s(RE2::DefaultOptions, RE2::UNANCHORED); + + ASSERT_EQ(s.Add("foo", NULL), 0); + ASSERT_EQ(s.Add("(", NULL), -1); + ASSERT_EQ(s.Add("foobar", NULL), 1); + ASSERT_EQ(s.Compile(), true); + + ASSERT_EQ(s.Match("foobar", NULL), true); + ASSERT_EQ(s.Match("obarfoobaroo", NULL), true); + ASSERT_EQ(s.Match("fooba", NULL), true); + ASSERT_EQ(s.Match("oobar", NULL), false); + + std::vector v; + ASSERT_EQ(s.Match("foobar", &v), true); + ASSERT_EQ(v.size(), 2); + ASSERT_EQ(v[0], 0); + ASSERT_EQ(v[1], 1); + + ASSERT_EQ(s.Match("obarfoobaroo", &v), true); + ASSERT_EQ(v.size(), 2); + ASSERT_EQ(v[0], 0); + ASSERT_EQ(v[1], 1); + + ASSERT_EQ(s.Match("fooba", &v), true); + ASSERT_EQ(v.size(), 1); + ASSERT_EQ(v[0], 0); + + ASSERT_EQ(s.Match("oobar", &v), false); + ASSERT_EQ(v.size(), 0); +} + +TEST(Set, UnanchoredDollar) { + RE2::Set s(RE2::DefaultOptions, RE2::UNANCHORED); + + ASSERT_EQ(s.Add("foo$", NULL), 0); + ASSERT_EQ(s.Compile(), true); + + ASSERT_EQ(s.Match("foo", NULL), true); + ASSERT_EQ(s.Match("foobar", NULL), false); + + std::vector v; + ASSERT_EQ(s.Match("foo", &v), true); + ASSERT_EQ(v.size(), 1); + ASSERT_EQ(v[0], 0); + + ASSERT_EQ(s.Match("foobar", &v), false); + ASSERT_EQ(v.size(), 0); +} + +TEST(Set, UnanchoredWordBoundary) { + RE2::Set s(RE2::DefaultOptions, RE2::UNANCHORED); + + ASSERT_EQ(s.Add("foo\\b", NULL), 0); + ASSERT_EQ(s.Compile(), true); + + ASSERT_EQ(s.Match("foo", NULL), true); + ASSERT_EQ(s.Match("foobar", NULL), false); + ASSERT_EQ(s.Match("foo bar", NULL), true); + + std::vector v; + ASSERT_EQ(s.Match("foo", &v), true); + ASSERT_EQ(v.size(), 1); + ASSERT_EQ(v[0], 0); + + ASSERT_EQ(s.Match("foobar", &v), false); + ASSERT_EQ(v.size(), 0); + + ASSERT_EQ(s.Match("foo bar", &v), true); + ASSERT_EQ(v.size(), 1); + ASSERT_EQ(v[0], 0); +} + +TEST(Set, Anchored) { + RE2::Set s(RE2::DefaultOptions, RE2::ANCHOR_BOTH); + + ASSERT_EQ(s.Add("foo", NULL), 0); + ASSERT_EQ(s.Add("(", NULL), -1); + ASSERT_EQ(s.Add("bar", NULL), 1); + ASSERT_EQ(s.Compile(), true); + + ASSERT_EQ(s.Match("foobar", NULL), false); + ASSERT_EQ(s.Match("fooba", NULL), false); + ASSERT_EQ(s.Match("oobar", NULL), false); + ASSERT_EQ(s.Match("foo", NULL), true); + ASSERT_EQ(s.Match("bar", NULL), true); + + std::vector v; + ASSERT_EQ(s.Match("foobar", &v), false); + ASSERT_EQ(v.size(), 0); + + ASSERT_EQ(s.Match("fooba", &v), false); + ASSERT_EQ(v.size(), 0); + + ASSERT_EQ(s.Match("oobar", &v), false); + ASSERT_EQ(v.size(), 0); + + ASSERT_EQ(s.Match("foo", &v), true); + ASSERT_EQ(v.size(), 1); + ASSERT_EQ(v[0], 0); + + ASSERT_EQ(s.Match("bar", &v), true); + ASSERT_EQ(v.size(), 1); + ASSERT_EQ(v[0], 1); +} + +TEST(Set, EmptyUnanchored) { + RE2::Set s(RE2::DefaultOptions, RE2::UNANCHORED); + + ASSERT_EQ(s.Compile(), true); + + ASSERT_EQ(s.Match("", NULL), false); + ASSERT_EQ(s.Match("foobar", NULL), false); + + std::vector v; + ASSERT_EQ(s.Match("", &v), false); + ASSERT_EQ(v.size(), 0); + + ASSERT_EQ(s.Match("foobar", &v), false); + ASSERT_EQ(v.size(), 0); +} + +TEST(Set, EmptyAnchored) { + RE2::Set s(RE2::DefaultOptions, RE2::ANCHOR_BOTH); + + ASSERT_EQ(s.Compile(), true); + + ASSERT_EQ(s.Match("", NULL), false); + ASSERT_EQ(s.Match("foobar", NULL), false); + + std::vector v; + ASSERT_EQ(s.Match("", &v), false); + ASSERT_EQ(v.size(), 0); + + ASSERT_EQ(s.Match("foobar", &v), false); + ASSERT_EQ(v.size(), 0); +} + +TEST(Set, Prefix) { + RE2::Set s(RE2::DefaultOptions, RE2::ANCHOR_BOTH); + + ASSERT_EQ(s.Add("/prefix/\\d*", NULL), 0); + ASSERT_EQ(s.Compile(), true); + + ASSERT_EQ(s.Match("/prefix", NULL), false); + ASSERT_EQ(s.Match("/prefix/", NULL), true); + ASSERT_EQ(s.Match("/prefix/42", NULL), true); + + std::vector v; + ASSERT_EQ(s.Match("/prefix", &v), false); + ASSERT_EQ(v.size(), 0); + + ASSERT_EQ(s.Match("/prefix/", &v), true); + ASSERT_EQ(v.size(), 1); + ASSERT_EQ(v[0], 0); + + ASSERT_EQ(s.Match("/prefix/42", &v), true); + ASSERT_EQ(v.size(), 1); + ASSERT_EQ(v[0], 0); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/simplify_test.cc b/extern/re2/re2/testing/simplify_test.cc new file mode 100644 index 0000000000..9dcd4ac6b2 --- /dev/null +++ b/extern/re2/re2/testing/simplify_test.cc @@ -0,0 +1,273 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test simplify.cc. + +#include +#include + +#include "util/test.h" +#include "util/logging.h" +#include "re2/regexp.h" + +namespace re2 { + +struct Test { + const char* regexp; + const char* simplified; +}; + +static Test tests[] = { + // Already-simple constructs + { "a", "a" }, + { "ab", "ab" }, + { "a|b", "[a-b]" }, + { "ab|cd", "ab|cd" }, + { "(ab)*", "(ab)*" }, + { "(ab)+", "(ab)+" }, + { "(ab)?", "(ab)?" }, + { ".", "." }, + { "^", "^" }, + { "$", "$" }, + { "[ac]", "[ac]" }, + { "[^ac]", "[^ac]" }, + + // Posix character classes + { "[[:alnum:]]", "[0-9A-Za-z]" }, + { "[[:alpha:]]", "[A-Za-z]" }, + { "[[:blank:]]", "[\\t ]" }, + { "[[:cntrl:]]", "[\\x00-\\x1f\\x7f]" }, + { "[[:digit:]]", "[0-9]" }, + { "[[:graph:]]", "[!-~]" }, + { "[[:lower:]]", "[a-z]" }, + { "[[:print:]]", "[ -~]" }, + { "[[:punct:]]", "[!-/:-@\\[-`{-~]" }, + { "[[:space:]]" , "[\\t-\\r ]" }, + { "[[:upper:]]", "[A-Z]" }, + { "[[:xdigit:]]", "[0-9A-Fa-f]" }, + + // Perl character classes + { "\\d", "[0-9]" }, + { "\\s", "[\\t-\\n\\f-\\r ]" }, + { "\\w", "[0-9A-Z_a-z]" }, + { "\\D", "[^0-9]" }, + { "\\S", "[^\\t-\\n\\f-\\r ]" }, + { "\\W", "[^0-9A-Z_a-z]" }, + { "[\\d]", "[0-9]" }, + { "[\\s]", "[\\t-\\n\\f-\\r ]" }, + { "[\\w]", "[0-9A-Z_a-z]" }, + { "[\\D]", "[^0-9]" }, + { "[\\S]", "[^\\t-\\n\\f-\\r ]" }, + { "[\\W]", "[^0-9A-Z_a-z]" }, + + // Posix repetitions + { "a{1}", "a" }, + { "a{2}", "aa" }, + { "a{5}", "aaaaa" }, + { "a{0,1}", "a?" }, + // The next three are illegible because Simplify inserts (?:) + // parens instead of () parens to avoid creating extra + // captured subexpressions. The comments show a version fewer parens. + { "(a){0,2}", "(?:(a)(a)?)?" }, // (aa?)? + { "(a){0,4}", "(?:(a)(?:(a)(?:(a)(a)?)?)?)?" }, // (a(a(aa?)?)?)? + { "(a){2,6}", "(a)(a)(?:(a)(?:(a)(?:(a)(a)?)?)?)?" }, // aa(a(a(aa?)?)?)? + { "a{0,2}", "(?:aa?)?" }, // (aa?)? + { "a{0,4}", "(?:a(?:a(?:aa?)?)?)?" }, // (a(a(aa?)?)?)? + { "a{2,6}", "aa(?:a(?:a(?:aa?)?)?)?" }, // aa(a(a(aa?)?)?)? + { "a{0,}", "a*" }, + { "a{1,}", "a+" }, + { "a{2,}", "aa+" }, + { "a{5,}", "aaaaa+" }, + + // Test that operators simplify their arguments. + // (Simplify used to not simplify arguments to a {} repeat.) + { "(?:a{1,}){1,}", "a+" }, + { "(a{1,}b{1,})", "(a+b+)" }, + { "a{1,}|b{1,}", "a+|b+" }, + { "(?:a{1,})*", "(?:a+)*" }, + { "(?:a{1,})+", "a+" }, + { "(?:a{1,})?", "(?:a+)?" }, + { "a{0}", "" }, + + // Character class simplification + { "[ab]", "[a-b]" }, + { "[a-za-za-z]", "[a-z]" }, + { "[A-Za-zA-Za-z]", "[A-Za-z]" }, + { "[ABCDEFGH]", "[A-H]" }, + { "[AB-CD-EF-GH]", "[A-H]" }, + { "[W-ZP-XE-R]", "[E-Z]" }, + { "[a-ee-gg-m]", "[a-m]" }, + { "[a-ea-ha-m]", "[a-m]" }, + { "[a-ma-ha-e]", "[a-m]" }, + { "[a-zA-Z0-9 -~]", "[ -~]" }, + + // Empty character classes + { "[^[:cntrl:][:^cntrl:]]", "[^\\x00-\\x{10ffff}]" }, + + // Full character classes + { "[[:cntrl:][:^cntrl:]]", "." }, + + // Unicode case folding. + { "(?i)A", "[Aa]" }, + { "(?i)a", "[Aa]" }, + { "(?i)K", "[Kk\\x{212a}]" }, + { "(?i)k", "[Kk\\x{212a}]" }, + { "(?i)\\x{212a}", "[Kk\\x{212a}]" }, + { "(?i)[a-z]", "[A-Za-z\\x{17f}\\x{212a}]" }, + { "(?i)[\\x00-\\x{FFFD}]", "[\\x00-\\x{fffd}]" }, + { "(?i)[\\x00-\\x{10ffff}]", "." }, + + // Empty string as a regular expression. + // Empty string must be preserved inside parens in order + // to make submatches work right, so these are less + // interesting than they used to be. ToString inserts + // explicit (?:) in place of non-parenthesized empty strings, + // to make them easier to spot for other parsers. + { "(a|b|)", "([a-b]|(?:))" }, + { "(|)", "((?:)|(?:))" }, + { "a()", "a()" }, + { "(()|())", "(()|())" }, + { "(a|)", "(a|(?:))" }, + { "ab()cd()", "ab()cd()" }, + { "()", "()" }, + { "()*", "()*" }, + { "()+", "()+" }, + { "()?" , "()?" }, + { "(){0}", "" }, + { "(){1}", "()" }, + { "(){1,}", "()+" }, + { "(){0,2}", "(?:()()?)?" }, + + // Test that coalescing occurs and that the resulting repeats are simplified. + // Two-op combinations of *, +, ?, {n}, {n,} and {n,m} with a literal: + { "a*a*", "a*" }, + { "a*a+", "a+" }, + { "a*a?", "a*" }, + { "a*a{2}", "aa+" }, + { "a*a{2,}", "aa+" }, + { "a*a{2,3}", "aa+" }, + { "a+a*", "a+" }, + { "a+a+", "aa+" }, + { "a+a?", "a+" }, + { "a+a{2}", "aaa+" }, + { "a+a{2,}", "aaa+" }, + { "a+a{2,3}", "aaa+" }, + { "a?a*", "a*" }, + { "a?a+", "a+" }, + { "a?a?", "(?:aa?)?" }, + { "a?a{2}", "aaa?" }, + { "a?a{2,}", "aa+" }, + { "a?a{2,3}", "aa(?:aa?)?" }, + { "a{2}a*", "aa+" }, + { "a{2}a+", "aaa+" }, + { "a{2}a?", "aaa?" }, + { "a{2}a{2}", "aaaa" }, + { "a{2}a{2,}", "aaaa+" }, + { "a{2}a{2,3}", "aaaaa?" }, + { "a{2,}a*", "aa+" }, + { "a{2,}a+", "aaa+" }, + { "a{2,}a?", "aa+" }, + { "a{2,}a{2}", "aaaa+" }, + { "a{2,}a{2,}", "aaaa+" }, + { "a{2,}a{2,3}", "aaaa+" }, + { "a{2,3}a*", "aa+" }, + { "a{2,3}a+", "aaa+" }, + { "a{2,3}a?", "aa(?:aa?)?" }, + { "a{2,3}a{2}", "aaaaa?" }, + { "a{2,3}a{2,}", "aaaa+" }, + { "a{2,3}a{2,3}", "aaaa(?:aa?)?" }, + // With a char class, any char and any byte: + { "\\d*\\d*", "[0-9]*" }, + { ".*.*", ".*" }, + { "\\C*\\C*", "\\C*" }, + // FoldCase works, but must be consistent: + { "(?i)A*a*", "[Aa]*" }, + { "(?i)a+A+", "[Aa][Aa]+" }, + { "(?i)A*(?-i)a*", "[Aa]*a*" }, + { "(?i)a+(?-i)A+", "[Aa]+A+" }, + // NonGreedy works, but must be consistent: + { "a*?a*?", "a*?" }, + { "a+?a+?", "aa+?" }, + { "a*?a*", "a*?a*" }, + { "a+a+?", "a+a+?" }, + // The second element is the literal, char class, any char or any byte: + { "a*a", "a+" }, + { "\\d*\\d", "[0-9]+" }, + { ".*.", ".+" }, + { "\\C*\\C", "\\C+" }, + // FoldCase works, but must be consistent: + { "(?i)A*a", "[Aa]+" }, + { "(?i)a+A", "[Aa][Aa]+" }, + { "(?i)A*(?-i)a", "[Aa]*a" }, + { "(?i)a+(?-i)A", "[Aa]+A" }, + // The second element is a literal string that begins with the literal: + { "a*aa", "aa+" }, + { "a*aab", "aa+b" }, + // FoldCase works, but must be consistent: + { "(?i)a*aa", "[Aa][Aa]+" }, + { "(?i)a*aab", "[Aa][Aa]+[Bb]" }, + { "(?i)a*(?-i)aa", "[Aa]*aa" }, + { "(?i)a*(?-i)aab", "[Aa]*aab" }, + // Negative tests with mismatching ops: + { "a*b*", "a*b*" }, + { "\\d*\\D*", "[0-9]*[^0-9]*" }, + { "a+b", "a+b" }, + { "\\d+\\D", "[0-9]+[^0-9]" }, + { "a?bb", "a?bb" }, + // Negative tests with capturing groups: + { "(a*)a*", "(a*)a*" }, + { "a+(a)", "a+(a)" }, + { "(a?)(aa)", "(a?)(aa)" }, + // Just for fun: + { "aa*aa+aa?aa{2}aaa{2,}aaa{2,3}a", "aaaaaaaaaaaaaaaa+" }, + + // During coalescing, the child of the repeat changes, so we build a new + // repeat. The new repeat must have the min and max of the old repeat. + // Failure to copy them results in min=0 and max=0 -> empty match. + { "(?:a*aab){2}", "aa+baa+b" }, + + // During coalescing, the child of the capture changes, so we build a new + // capture. The new capture must have the cap of the old capture. + // Failure to copy it results in cap=0 -> ToString() logs a fatal error. + { "(a*aab)", "(aa+b)" }, + + // Test squashing of **, ++, ?? et cetera. + { "(?:(?:a){0,}){0,}", "a*" }, + { "(?:(?:a){1,}){1,}", "a+" }, + { "(?:(?:a){0,1}){0,1}", "a?" }, + { "(?:(?:a){0,}){1,}", "a*" }, + { "(?:(?:a){0,}){0,1}", "a*" }, + { "(?:(?:a){1,}){0,}", "a*" }, + { "(?:(?:a){1,}){0,1}", "a*" }, + { "(?:(?:a){0,1}){0,}", "a*" }, + { "(?:(?:a){0,1}){1,}", "a*" }, +}; + +TEST(TestSimplify, SimpleRegexps) { + for (size_t i = 0; i < arraysize(tests); i++) { + RegexpStatus status; + VLOG(1) << "Testing " << tests[i].regexp; + Regexp* re = Regexp::Parse(tests[i].regexp, + Regexp::MatchNL | (Regexp::LikePerl & + ~Regexp::OneLine), + &status); + ASSERT_TRUE(re != NULL) << " " << tests[i].regexp << " " << status.Text(); + Regexp* sre = re->Simplify(); + ASSERT_TRUE(sre != NULL); + + // Check that already-simple regexps don't allocate new ones. + if (strcmp(tests[i].regexp, tests[i].simplified) == 0) { + ASSERT_TRUE(re == sre) << " " << tests[i].regexp + << " " << re->ToString() << " " << sre->ToString(); + } + + EXPECT_EQ(tests[i].simplified, sre->ToString()) + << " " << tests[i].regexp << " " << sre->Dump(); + + re->Decref(); + sre->Decref(); + } +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/string_generator.cc b/extern/re2/re2/testing/string_generator.cc new file mode 100644 index 0000000000..030cc457e1 --- /dev/null +++ b/extern/re2/re2/testing/string_generator.cc @@ -0,0 +1,114 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// String generator: generates all possible strings of up to +// maxlen letters using the set of letters in alpha. +// Fetch strings using a Java-like Next()/HasNext() interface. + +#include +#include +#include +#include + +#include "util/test.h" +#include "util/logging.h" +#include "re2/testing/string_generator.h" + +namespace re2 { + +StringGenerator::StringGenerator(int maxlen, + const std::vector& alphabet) + : maxlen_(maxlen), alphabet_(alphabet), + generate_null_(false), + random_(false), nrandom_(0) { + + // Degenerate case: no letters, no non-empty strings. + if (alphabet_.empty()) + maxlen_ = 0; + + // Next() will return empty string (digits_ is empty). + hasnext_ = true; +} + +// Resets the string generator state to the beginning. +void StringGenerator::Reset() { + digits_.clear(); + hasnext_ = true; + random_ = false; + nrandom_ = 0; + generate_null_ = false; +} + +// Increments the big number in digits_, returning true if successful. +// Returns false if all the numbers have been used. +bool StringGenerator::IncrementDigits() { + // First try to increment the current number. + for (int i = static_cast(digits_.size()) - 1; i >= 0; i--) { + if (++digits_[i] < static_cast(alphabet_.size())) + return true; + digits_[i] = 0; + } + + // If that failed, make a longer number. + if (static_cast(digits_.size()) < maxlen_) { + digits_.push_back(0); + return true; + } + + return false; +} + +// Generates random digits_, return true if successful. +// Returns false if the random sequence is over. +bool StringGenerator::RandomDigits() { + if (--nrandom_ <= 0) + return false; + + std::uniform_int_distribution random_len(0, maxlen_); + std::uniform_int_distribution random_alphabet_index( + 0, static_cast(alphabet_.size()) - 1); + + // Pick length. + int len = random_len(rng_); + digits_.resize(len); + for (int i = 0; i < len; i++) + digits_[i] = random_alphabet_index(rng_); + return true; +} + +// Returns the next string in the iteration, which is the one +// currently described by digits_. Calls IncrementDigits +// after computing the string, so that it knows the answer +// for subsequent HasNext() calls. +const StringPiece& StringGenerator::Next() { + CHECK(hasnext_); + if (generate_null_) { + generate_null_ = false; + sp_ = StringPiece(); + return sp_; + } + s_.clear(); + for (size_t i = 0; i < digits_.size(); i++) { + s_ += alphabet_[digits_[i]]; + } + hasnext_ = random_ ? RandomDigits() : IncrementDigits(); + sp_ = s_; + return sp_; +} + +// Sets generator up to return n random strings. +void StringGenerator::Random(int32_t seed, int n) { + rng_.seed(seed); + + random_ = true; + nrandom_ = n; + hasnext_ = nrandom_ > 0; +} + +void StringGenerator::GenerateNULL() { + generate_null_ = true; + hasnext_ = true; +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/string_generator.h b/extern/re2/re2/testing/string_generator.h new file mode 100644 index 0000000000..6184176523 --- /dev/null +++ b/extern/re2/re2/testing/string_generator.h @@ -0,0 +1,63 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_TESTING_STRING_GENERATOR_H_ +#define RE2_TESTING_STRING_GENERATOR_H_ + +// String generator: generates all possible strings of up to +// maxlen letters using the set of letters in alpha. +// Fetch strings using a Java-like Next()/HasNext() interface. + +#include +#include +#include +#include + +#include "util/util.h" +#include "re2/stringpiece.h" + +namespace re2 { + +class StringGenerator { + public: + StringGenerator(int maxlen, const std::vector& alphabet); + ~StringGenerator() {} + + const StringPiece& Next(); + bool HasNext() { return hasnext_; } + + // Resets generator to start sequence over. + void Reset(); + + // Causes generator to emit random strings for next n calls to Next(). + void Random(int32_t seed, int n); + + // Causes generator to emit a NULL as the next call. + void GenerateNULL(); + + private: + bool IncrementDigits(); + bool RandomDigits(); + + // Global state. + int maxlen_; // Maximum length string to generate. + std::vector alphabet_; // Alphabet, one string per letter. + + // Iteration state. + StringPiece sp_; // Last StringPiece returned by Next(). + std::string s_; // String data in last StringPiece returned by Next(). + bool hasnext_; // Whether Next() can be called again. + std::vector digits_; // Alphabet indices for next string. + bool generate_null_; // Whether to generate a NULL StringPiece next. + bool random_; // Whether generated strings are random. + int nrandom_; // Number of random strings left to generate. + std::minstd_rand0 rng_; // Random number generator. + + StringGenerator(const StringGenerator&) = delete; + StringGenerator& operator=(const StringGenerator&) = delete; +}; + +} // namespace re2 + +#endif // RE2_TESTING_STRING_GENERATOR_H_ diff --git a/extern/re2/re2/testing/string_generator_test.cc b/extern/re2/re2/testing/string_generator_test.cc new file mode 100644 index 0000000000..d0f84f4085 --- /dev/null +++ b/extern/re2/re2/testing/string_generator_test.cc @@ -0,0 +1,110 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test StringGenerator. + +#include +#include + +#include "util/test.h" +#include "util/utf.h" +#include "re2/testing/string_generator.h" +#include "re2/testing/regexp_generator.h" + +namespace re2 { + +// Returns i to the e. +static int64_t IntegerPower(int i, int e) { + int64_t p = 1; + while (e-- > 0) + p *= i; + return p; +} + +// Checks that for given settings of the string generator: +// * it generates strings that are non-decreasing in length. +// * strings of the same length are sorted in alphabet order. +// * it doesn't generate the same string twice. +// * it generates the right number of strings. +// +// If all of these hold, the StringGenerator is behaving. +// Assumes that the alphabet is sorted, so that the generated +// strings can just be compared lexicographically. +static void RunTest(int len, const std::string& alphabet, bool donull) { + StringGenerator g(len, Explode(alphabet)); + + int n = 0; + int last_l = -1; + std::string last_s; + + if (donull) { + g.GenerateNULL(); + EXPECT_TRUE(g.HasNext()); + StringPiece sp = g.Next(); + EXPECT_EQ(sp.data(), static_cast(NULL)); + EXPECT_EQ(sp.size(), 0); + } + + while (g.HasNext()) { + std::string s = std::string(g.Next()); + n++; + + // Check that all characters in s appear in alphabet. + for (const char *p = s.c_str(); *p != '\0'; ) { + Rune r; + p += chartorune(&r, p); + EXPECT_TRUE(utfrune(alphabet.c_str(), r) != NULL); + } + + // Check that string is properly ordered w.r.t. previous string. + int l = utflen(s.c_str()); + EXPECT_LE(l, len); + if (last_l < l) { + last_l = l; + } else { + EXPECT_EQ(last_l, l); + EXPECT_LT(last_s, s); + } + last_s = s; + } + + // Check total string count. + int64_t m = 0; + int alpha = utflen(alphabet.c_str()); + if (alpha == 0) // Degenerate case. + len = 0; + for (int i = 0; i <= len; i++) + m += IntegerPower(alpha, i); + EXPECT_EQ(n, m); +} + +TEST(StringGenerator, NoLength) { + RunTest(0, "abc", false); +} + +TEST(StringGenerator, NoLengthNoAlphabet) { + RunTest(0, "", false); +} + +TEST(StringGenerator, NoAlphabet) { + RunTest(5, "", false); +} + +TEST(StringGenerator, Simple) { + RunTest(3, "abc", false); +} + +TEST(StringGenerator, UTF8) { + RunTest(4, "abc\xE2\x98\xBA", false); +} + +TEST(StringGenerator, GenNULL) { + RunTest(0, "abc", true); + RunTest(0, "", true); + RunTest(5, "", true); + RunTest(3, "abc", true); + RunTest(4, "abc\xE2\x98\xBA", true); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/tester.cc b/extern/re2/re2/testing/tester.cc new file mode 100644 index 0000000000..d676d9a74f --- /dev/null +++ b/extern/re2/re2/testing/tester.cc @@ -0,0 +1,669 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Regular expression engine tester -- test all the implementations against each other. + +#include +#include +#include +#include + +#include "util/util.h" +#include "util/flags.h" +#include "util/logging.h" +#include "util/strutil.h" +#include "re2/testing/tester.h" +#include "re2/prog.h" +#include "re2/re2.h" +#include "re2/regexp.h" + +DEFINE_bool(dump_prog, false, "dump regexp program"); +DEFINE_bool(log_okay, false, "log successful runs"); +DEFINE_bool(dump_rprog, false, "dump reversed regexp program"); + +DEFINE_int32(max_regexp_failures, 100, + "maximum number of regexp test failures (-1 = unlimited)"); + +DEFINE_string(regexp_engines, "", "pattern to select regexp engines to test"); + +namespace re2 { + +enum { + kMaxSubmatch = 1+16, // $0...$16 +}; + +const char* engine_names[kEngineMax] = { + "Backtrack", + "NFA", + "DFA", + "DFA1", + "OnePass", + "BitState", + "RE2", + "RE2a", + "RE2b", + "PCRE", +}; + +// Returns the name of the engine. +static const char* EngineName(Engine e) { + CHECK_GE(e, 0); + CHECK_LT(e, arraysize(engine_names)); + CHECK(engine_names[e] != NULL); + return engine_names[e]; +} + +// Returns bit mask of engines to use. +static uint32_t Engines() { + static bool did_parse = false; + static uint32_t cached_engines = 0; + + if (did_parse) + return cached_engines; + + if (FLAGS_regexp_engines.empty()) { + cached_engines = ~0; + } else { + for (Engine i = static_cast(0); i < kEngineMax; i++) + if (FLAGS_regexp_engines.find(EngineName(i)) != std::string::npos) + cached_engines |= 1<(0); i < kEngineMax; i++) { + if (cached_engines & (1<= 0x80) bytes. +static bool NonASCII(const StringPiece& text) { + for (size_t i = 0; i < text.size(); i++) + if ((uint8_t)text[i] >= 0x80) + return true; + return false; +} + +// Returns string representation of match kind. +static std::string FormatKind(Prog::MatchKind kind) { + switch (kind) { + case Prog::kFullMatch: + return "full match"; + case Prog::kLongestMatch: + return "longest match"; + case Prog::kFirstMatch: + return "first match"; + case Prog::kManyMatch: + return "many match"; + } + return "???"; +} + +// Returns string representation of anchor kind. +static std::string FormatAnchor(Prog::Anchor anchor) { + switch (anchor) { + case Prog::kAnchored: + return "anchored"; + case Prog::kUnanchored: + return "unanchored"; + } + return "???"; +} + +struct ParseMode { + Regexp::ParseFlags parse_flags; + std::string desc; +}; + +static const Regexp::ParseFlags single_line = + Regexp::LikePerl; +static const Regexp::ParseFlags multi_line = + static_cast(Regexp::LikePerl & ~Regexp::OneLine); + +static ParseMode parse_modes[] = { + { single_line, "single-line" }, + { single_line|Regexp::Latin1, "single-line, latin1" }, + { multi_line, "multiline" }, + { multi_line|Regexp::NonGreedy, "multiline, nongreedy" }, + { multi_line|Regexp::Latin1, "multiline, latin1" }, +}; + +static std::string FormatMode(Regexp::ParseFlags flags) { + for (size_t i = 0; i < arraysize(parse_modes); i++) + if (parse_modes[i].parse_flags == flags) + return parse_modes[i].desc; + return StringPrintf("%#x", static_cast(flags)); +} + +// Constructs and saves all the matching engines that +// will be required for the given tests. +TestInstance::TestInstance(const StringPiece& regexp_str, Prog::MatchKind kind, + Regexp::ParseFlags flags) + : regexp_str_(regexp_str), + kind_(kind), + flags_(flags), + error_(false), + regexp_(NULL), + num_captures_(0), + prog_(NULL), + rprog_(NULL), + re_(NULL), + re2_(NULL) { + + VLOG(1) << CEscape(regexp_str); + + // Compile regexp to prog. + // Always required - needed for backtracking (reference implementation). + RegexpStatus status; + regexp_ = Regexp::Parse(regexp_str, flags, &status); + if (regexp_ == NULL) { + LOG(INFO) << "Cannot parse: " << CEscape(regexp_str_) + << " mode: " << FormatMode(flags); + error_ = true; + return; + } + num_captures_ = regexp_->NumCaptures(); + prog_ = regexp_->CompileToProg(0); + if (prog_ == NULL) { + LOG(INFO) << "Cannot compile: " << CEscape(regexp_str_); + error_ = true; + return; + } + if (FLAGS_dump_prog) { + LOG(INFO) << "Prog for " + << " regexp " + << CEscape(regexp_str_) + << " (" << FormatKind(kind_) + << ", " << FormatMode(flags_) + << ")\n" + << prog_->Dump(); + } + + // Compile regexp to reversed prog. Only needed for DFA engines. + if (Engines() & ((1<CompileToReverseProg(0); + if (rprog_ == NULL) { + LOG(INFO) << "Cannot reverse compile: " << CEscape(regexp_str_); + error_ = true; + return; + } + if (FLAGS_dump_rprog) + LOG(INFO) << rprog_->Dump(); + } + + // Create re string that will be used for RE and RE2. + std::string re = std::string(regexp_str); + // Accomodate flags. + // Regexp::Latin1 will be accomodated below. + if (!(flags & Regexp::OneLine)) + re = "(?m)" + re; + if (flags & Regexp::NonGreedy) + re = "(?U)" + re; + if (flags & Regexp::DotNL) + re = "(?s)" + re; + + // Compile regexp to RE2. + if (Engines() & ((1<error().empty()) { + LOG(INFO) << "Cannot RE2: " << CEscape(re); + error_ = true; + return; + } + } + + // Compile regexp to RE. + // PCRE as exposed by the RE interface isn't always usable. + // 1. It disagrees about handling of empty-string reptitions + // like matching (a*)* against "b". PCRE treats the (a*) as + // occurring once, while we treat it as occurring not at all. + // 2. It treats $ as this weird thing meaning end of string + // or before the \n at the end of the string. + // 3. It doesn't implement POSIX leftmost-longest matching. + // 4. It lets \s match vertical tab. + // MimicsPCRE() detects 1 and 2. + if ((Engines() & (1<MimicsPCRE() && + kind_ != Prog::kLongestMatch) { + PCRE_Options o; + o.set_option(PCRE::UTF8); + if (flags & Regexp::Latin1) + o.set_option(PCRE::None); + // PCRE has interface bug keeping us from finding $0, so + // add one more layer of parens. + re_ = new PCRE("("+re+")", o); + if (!re_->error().empty()) { + LOG(INFO) << "Cannot PCRE: " << CEscape(re); + error_ = true; + return; + } + } +} + +TestInstance::~TestInstance() { + if (regexp_) + regexp_->Decref(); + delete prog_; + delete rprog_; + delete re_; + delete re2_; +} + +// Runs a single search using the named engine type. +// This interface hides all the irregularities of the various +// engine interfaces from the rest of this file. +void TestInstance::RunSearch(Engine type, + const StringPiece& orig_text, + const StringPiece& orig_context, + Prog::Anchor anchor, + Result* result) { + // Result is not trivial, so we cannot freely clear it with memset(3), + // but zeroing objects like so is safe and expedient for our purposes. + memset(reinterpret_cast(result), 0, sizeof *result); + if (regexp_ == NULL) { + result->skipped = true; + return; + } + int nsubmatch = 1 + num_captures_; // NumCaptures doesn't count $0 + if (nsubmatch > kMaxSubmatch) + nsubmatch = kMaxSubmatch; + + StringPiece text = orig_text; + StringPiece context = orig_context; + + switch (type) { + default: + LOG(FATAL) << "Bad RunSearch type: " << (int)type; + + case kEngineBacktrack: + if (prog_ == NULL) { + result->skipped = true; + break; + } + result->matched = + prog_->UnsafeSearchBacktrack(text, context, anchor, kind_, + result->submatch, nsubmatch); + result->have_submatch = true; + break; + + case kEngineNFA: + if (prog_ == NULL) { + result->skipped = true; + break; + } + result->matched = + prog_->SearchNFA(text, context, anchor, kind_, + result->submatch, nsubmatch); + result->have_submatch = true; + break; + + case kEngineDFA: + if (prog_ == NULL) { + result->skipped = true; + break; + } + result->matched = prog_->SearchDFA(text, context, anchor, kind_, NULL, + &result->skipped, NULL); + break; + + case kEngineDFA1: + if (prog_ == NULL || rprog_ == NULL) { + result->skipped = true; + break; + } + result->matched = + prog_->SearchDFA(text, context, anchor, kind_, result->submatch, + &result->skipped, NULL); + // If anchored, no need for second run, + // but do it anyway to find more bugs. + if (result->matched) { + if (!rprog_->SearchDFA(result->submatch[0], context, + Prog::kAnchored, Prog::kLongestMatch, + result->submatch, + &result->skipped, NULL)) { + LOG(ERROR) << "Reverse DFA inconsistency: " + << CEscape(regexp_str_) + << " on " << CEscape(text); + result->matched = false; + } + } + result->have_submatch0 = true; + break; + + case kEngineOnePass: + if (prog_ == NULL || + !prog_->IsOnePass() || + anchor == Prog::kUnanchored || + nsubmatch > Prog::kMaxOnePassCapture) { + result->skipped = true; + break; + } + result->matched = prog_->SearchOnePass(text, context, anchor, kind_, + result->submatch, nsubmatch); + result->have_submatch = true; + break; + + case kEngineBitState: + if (prog_ == NULL || + !prog_->CanBitState()) { + result->skipped = true; + break; + } + result->matched = prog_->SearchBitState(text, context, anchor, kind_, + result->submatch, nsubmatch); + result->have_submatch = true; + break; + + case kEngineRE2: + case kEngineRE2a: + case kEngineRE2b: { + if (!re2_ || text.end() != context.end()) { + result->skipped = true; + break; + } + + RE2::Anchor re_anchor; + if (anchor == Prog::kAnchored) + re_anchor = RE2::ANCHOR_START; + else + re_anchor = RE2::UNANCHORED; + if (kind_ == Prog::kFullMatch) + re_anchor = RE2::ANCHOR_BOTH; + + result->matched = re2_->Match( + context, + static_cast(text.begin() - context.begin()), + static_cast(text.end() - context.begin()), + re_anchor, + result->submatch, + nsubmatch); + result->have_submatch = nsubmatch > 0; + break; + } + + case kEnginePCRE: { + if (!re_ || text.begin() != context.begin() || + text.end() != context.end()) { + result->skipped = true; + break; + } + + // In Perl/PCRE, \v matches any character considered vertical + // whitespace, not just vertical tab. Regexp::MimicsPCRE() is + // unable to handle all cases of this, unfortunately, so just + // catch them here. :( + if (regexp_str_.find("\\v") != StringPiece::npos && + (text.find('\n') != StringPiece::npos || + text.find('\f') != StringPiece::npos || + text.find('\r') != StringPiece::npos)) { + result->skipped = true; + break; + } + + // PCRE 8.34 or so started allowing vertical tab to match \s, + // following a change made in Perl 5.18. RE2 does not. + if ((regexp_str_.find("\\s") != StringPiece::npos || + regexp_str_.find("\\S") != StringPiece::npos) && + text.find('\v') != StringPiece::npos) { + result->skipped = true; + break; + } + + const PCRE::Arg **argptr = new const PCRE::Arg*[nsubmatch]; + PCRE::Arg *a = new PCRE::Arg[nsubmatch]; + for (int i = 0; i < nsubmatch; i++) { + a[i] = PCRE::Arg(&result->submatch[i]); + argptr[i] = &a[i]; + } + size_t consumed; + PCRE::Anchor pcre_anchor; + if (anchor == Prog::kAnchored) + pcre_anchor = PCRE::ANCHOR_START; + else + pcre_anchor = PCRE::UNANCHORED; + if (kind_ == Prog::kFullMatch) + pcre_anchor = PCRE::ANCHOR_BOTH; + re_->ClearHitLimit(); + result->matched = + re_->DoMatch(text, + pcre_anchor, + &consumed, + argptr, nsubmatch); + if (re_->HitLimit()) { + result->untrusted = true; + delete[] argptr; + delete[] a; + break; + } + result->have_submatch = true; + delete[] argptr; + delete[] a; + break; + } + } + + if (!result->matched) + memset(result->submatch, 0, sizeof result->submatch); +} + +// Checks whether r is okay given that correct is the right answer. +// Specifically, r's answers have to match (but it doesn't have to +// claim to have all the answers). +static bool ResultOkay(const Result& r, const Result& correct) { + if (r.skipped) + return true; + if (r.matched != correct.matched) + return false; + if (r.have_submatch || r.have_submatch0) { + for (int i = 0; i < kMaxSubmatch; i++) { + if (correct.submatch[i].begin() != r.submatch[i].begin() || + correct.submatch[i].size() != r.submatch[i].size()) + return false; + if (!r.have_submatch) + break; + } + } + return true; +} + +// Runs a single test. +bool TestInstance::RunCase(const StringPiece& text, const StringPiece& context, + Prog::Anchor anchor) { + // Backtracking is the gold standard. + Result correct; + RunSearch(kEngineBacktrack, text, context, anchor, &correct); + if (correct.skipped) { + if (regexp_ == NULL) + return true; + LOG(ERROR) << "Skipped backtracking! " << CEscape(regexp_str_) + << " " << FormatMode(flags_); + return false; + } + VLOG(1) << "Try: regexp " << CEscape(regexp_str_) + << " text " << CEscape(text) + << " (" << FormatKind(kind_) + << ", " << FormatAnchor(anchor) + << ", " << FormatMode(flags_) + << ")"; + + // Compare the others. + bool all_okay = true; + for (Engine i = kEngineBacktrack+1; i < kEngineMax; i++) { + if (!(Engines() & (1< 0 && --FLAGS_max_regexp_failures == 0) + LOG(QFATAL) << "Too many regexp failures."; + } + + return all_okay; +} + +void TestInstance::LogMatch(const char* prefix, Engine e, + const StringPiece& text, const StringPiece& context, + Prog::Anchor anchor) { + LOG(INFO) << prefix + << EngineName(e) + << " regexp " + << CEscape(regexp_str_) + << " " + << CEscape(regexp_->ToString()) + << " text " + << CEscape(text) + << " (" + << text.begin() - context.begin() + << "," + << text.end() - context.begin() + << ") of context " + << CEscape(context) + << " (" << FormatKind(kind_) + << ", " << FormatAnchor(anchor) + << ", " << FormatMode(flags_) + << ")"; +} + +static Prog::MatchKind kinds[] = { + Prog::kFirstMatch, + Prog::kLongestMatch, + Prog::kFullMatch, +}; + +// Test all possible match kinds and parse modes. +Tester::Tester(const StringPiece& regexp) { + error_ = false; + for (size_t i = 0; i < arraysize(kinds); i++) { + for (size_t j = 0; j < arraysize(parse_modes); j++) { + TestInstance* t = new TestInstance(regexp, kinds[i], + parse_modes[j].parse_flags); + error_ |= t->error(); + v_.push_back(t); + } + } +} + +Tester::~Tester() { + for (size_t i = 0; i < v_.size(); i++) + delete v_[i]; +} + +bool Tester::TestCase(const StringPiece& text, const StringPiece& context, + Prog::Anchor anchor) { + bool okay = true; + for (size_t i = 0; i < v_.size(); i++) + okay &= (!v_[i]->error() && v_[i]->RunCase(text, context, anchor)); + return okay; +} + +static Prog::Anchor anchors[] = { + Prog::kAnchored, + Prog::kUnanchored +}; + +bool Tester::TestInput(const StringPiece& text) { + bool okay = TestInputInContext(text, text); + if (text.size() > 0) { + StringPiece sp; + sp = text; + sp.remove_prefix(1); + okay &= TestInputInContext(sp, text); + sp = text; + sp.remove_suffix(1); + okay &= TestInputInContext(sp, text); + } + return okay; +} + +bool Tester::TestInputInContext(const StringPiece& text, + const StringPiece& context) { + bool okay = true; + for (size_t i = 0; i < arraysize(anchors); i++) + okay &= TestCase(text, context, anchors[i]); + return okay; +} + +bool TestRegexpOnText(const StringPiece& regexp, + const StringPiece& text) { + Tester t(regexp); + return t.TestInput(text); +} + +} // namespace re2 diff --git a/extern/re2/re2/testing/tester.h b/extern/re2/re2/testing/tester.h new file mode 100644 index 0000000000..47d0c4304f --- /dev/null +++ b/extern/re2/re2/testing/tester.h @@ -0,0 +1,123 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_TESTING_TESTER_H_ +#define RE2_TESTING_TESTER_H_ + +// Comparative tester for regular expression matching. +// Checks all implementations against each other. + +#include + +#include "re2/stringpiece.h" +#include "re2/prog.h" +#include "re2/regexp.h" +#include "re2/re2.h" +#include "util/pcre.h" + +namespace re2 { + +// All the supported regexp engines. +enum Engine { + kEngineBacktrack = 0, // Prog::UnsafeSearchBacktrack + kEngineNFA, // Prog::SearchNFA + kEngineDFA, // Prog::SearchDFA, only ask whether it matched + kEngineDFA1, // Prog::SearchDFA, ask for match[0] + kEngineOnePass, // Prog::SearchOnePass, if applicable + kEngineBitState, // Prog::SearchBitState + kEngineRE2, // RE2, all submatches + kEngineRE2a, // RE2, only ask for match[0] + kEngineRE2b, // RE2, only ask whether it matched + kEnginePCRE, // PCRE (util/pcre.h) + + kEngineMax, +}; + +// Make normal math on the enum preserve the type. +// By default, C++ doesn't define ++ on enum, and e+1 has type int. +static inline void operator++(Engine& e, int unused) { + e = static_cast(e+1); +} + +static inline Engine operator+(Engine e, int i) { + return static_cast(static_cast(e)+i); +} + +// A TestInstance caches per-regexp state for a given +// regular expression in a given configuration +// (UTF-8 vs Latin1, longest vs first match, etc.). +class TestInstance { + public: + struct Result; + + TestInstance(const StringPiece& regexp, Prog::MatchKind kind, + Regexp::ParseFlags flags); + ~TestInstance(); + Regexp::ParseFlags flags() { return flags_; } + bool error() { return error_; } + + // Runs a single test case: search in text, which is in context, + // using the given anchoring. + bool RunCase(const StringPiece& text, const StringPiece& context, + Prog::Anchor anchor); + + private: + // Runs a single search using the named engine type. + void RunSearch(Engine type, + const StringPiece& text, const StringPiece& context, + Prog::Anchor anchor, + Result *result); + + void LogMatch(const char* prefix, Engine e, const StringPiece& text, + const StringPiece& context, Prog::Anchor anchor); + + const StringPiece regexp_str_; // regexp being tested + Prog::MatchKind kind_; // kind of match + Regexp::ParseFlags flags_; // flags for parsing regexp_str_ + bool error_; // error during constructor? + + Regexp* regexp_; // parsed regexp + int num_captures_; // regexp_->NumCaptures() cached + Prog* prog_; // compiled program + Prog* rprog_; // compiled reverse program + PCRE* re_; // PCRE implementation + RE2* re2_; // RE2 implementation + + TestInstance(const TestInstance&) = delete; + TestInstance& operator=(const TestInstance&) = delete; +}; + +// A group of TestInstances for all possible configurations. +class Tester { + public: + explicit Tester(const StringPiece& regexp); + ~Tester(); + + bool error() { return error_; } + + // Runs a single test case: search in text, which is in context, + // using the given anchoring. + bool TestCase(const StringPiece& text, const StringPiece& context, + Prog::Anchor anchor); + + // Run TestCase(text, text, anchor) for all anchoring modes. + bool TestInput(const StringPiece& text); + + // Run TestCase(text, context, anchor) for all anchoring modes. + bool TestInputInContext(const StringPiece& text, const StringPiece& context); + + private: + bool error_; + std::vector v_; + + Tester(const Tester&) = delete; + Tester& operator=(const Tester&) = delete; +}; + +// Run all possible tests using regexp and text. +bool TestRegexpOnText(const StringPiece& regexp, const StringPiece& text); + +} // namespace re2 + +#endif // RE2_TESTING_TESTER_H_ diff --git a/extern/re2/re2/tostring.cc b/extern/re2/re2/tostring.cc new file mode 100644 index 0000000000..4545a92ddb --- /dev/null +++ b/extern/re2/re2/tostring.cc @@ -0,0 +1,351 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Format a regular expression structure as a string. +// Tested by parse_test.cc + +#include +#include + +#include "util/util.h" +#include "util/logging.h" +#include "util/strutil.h" +#include "util/utf.h" +#include "re2/regexp.h" +#include "re2/walker-inl.h" + +namespace re2 { + +enum { + PrecAtom, + PrecUnary, + PrecConcat, + PrecAlternate, + PrecEmpty, + PrecParen, + PrecToplevel, +}; + +// Helper function. See description below. +static void AppendCCRange(std::string* t, Rune lo, Rune hi); + +// Walker to generate string in s_. +// The arg pointers are actually integers giving the +// context precedence. +// The child_args are always NULL. +class ToStringWalker : public Regexp::Walker { + public: + explicit ToStringWalker(std::string* t) : t_(t) {} + + virtual int PreVisit(Regexp* re, int parent_arg, bool* stop); + virtual int PostVisit(Regexp* re, int parent_arg, int pre_arg, + int* child_args, int nchild_args); + virtual int ShortVisit(Regexp* re, int parent_arg) { + return 0; + } + + private: + std::string* t_; // The string the walker appends to. + + ToStringWalker(const ToStringWalker&) = delete; + ToStringWalker& operator=(const ToStringWalker&) = delete; +}; + +std::string Regexp::ToString() { + std::string t; + ToStringWalker w(&t); + w.WalkExponential(this, PrecToplevel, 100000); + if (w.stopped_early()) + t += " [truncated]"; + return t; +} + +#define ToString DontCallToString // Avoid accidental recursion. + +// Visits re before children are processed. +// Appends ( if needed and passes new precedence to children. +int ToStringWalker::PreVisit(Regexp* re, int parent_arg, bool* stop) { + int prec = parent_arg; + int nprec = PrecAtom; + + switch (re->op()) { + case kRegexpNoMatch: + case kRegexpEmptyMatch: + case kRegexpLiteral: + case kRegexpAnyChar: + case kRegexpAnyByte: + case kRegexpBeginLine: + case kRegexpEndLine: + case kRegexpBeginText: + case kRegexpEndText: + case kRegexpWordBoundary: + case kRegexpNoWordBoundary: + case kRegexpCharClass: + case kRegexpHaveMatch: + nprec = PrecAtom; + break; + + case kRegexpConcat: + case kRegexpLiteralString: + if (prec < PrecConcat) + t_->append("(?:"); + nprec = PrecConcat; + break; + + case kRegexpAlternate: + if (prec < PrecAlternate) + t_->append("(?:"); + nprec = PrecAlternate; + break; + + case kRegexpCapture: + t_->append("("); + if (re->cap() == 0) + LOG(DFATAL) << "kRegexpCapture cap() == 0"; + if (re->name()) { + t_->append("?P<"); + t_->append(*re->name()); + t_->append(">"); + } + nprec = PrecParen; + break; + + case kRegexpStar: + case kRegexpPlus: + case kRegexpQuest: + case kRegexpRepeat: + if (prec < PrecUnary) + t_->append("(?:"); + // The subprecedence here is PrecAtom instead of PrecUnary + // because PCRE treats two unary ops in a row as a parse error. + nprec = PrecAtom; + break; + } + + return nprec; +} + +static void AppendLiteral(std::string *t, Rune r, bool foldcase) { + if (r != 0 && r < 0x80 && strchr("(){}[]*+?|.^$\\", r)) { + t->append(1, '\\'); + t->append(1, static_cast(r)); + } else if (foldcase && 'a' <= r && r <= 'z') { + r -= 'a' - 'A'; + t->append(1, '['); + t->append(1, static_cast(r)); + t->append(1, static_cast(r) + 'a' - 'A'); + t->append(1, ']'); + } else { + AppendCCRange(t, r, r); + } +} + +// Visits re after children are processed. +// For childless regexps, all the work is done here. +// For regexps with children, append any unary suffixes or ). +int ToStringWalker::PostVisit(Regexp* re, int parent_arg, int pre_arg, + int* child_args, int nchild_args) { + int prec = parent_arg; + switch (re->op()) { + case kRegexpNoMatch: + // There's no simple symbol for "no match", but + // [^0-Runemax] excludes everything. + t_->append("[^\\x00-\\x{10ffff}]"); + break; + + case kRegexpEmptyMatch: + // Append (?:) to make empty string visible, + // unless this is already being parenthesized. + if (prec < PrecEmpty) + t_->append("(?:)"); + break; + + case kRegexpLiteral: + AppendLiteral(t_, re->rune(), + (re->parse_flags() & Regexp::FoldCase) != 0); + break; + + case kRegexpLiteralString: + for (int i = 0; i < re->nrunes(); i++) + AppendLiteral(t_, re->runes()[i], + (re->parse_flags() & Regexp::FoldCase) != 0); + if (prec < PrecConcat) + t_->append(")"); + break; + + case kRegexpConcat: + if (prec < PrecConcat) + t_->append(")"); + break; + + case kRegexpAlternate: + // Clumsy but workable: the children all appended | + // at the end of their strings, so just remove the last one. + if ((*t_)[t_->size()-1] == '|') + t_->erase(t_->size()-1); + else + LOG(DFATAL) << "Bad final char: " << t_; + if (prec < PrecAlternate) + t_->append(")"); + break; + + case kRegexpStar: + t_->append("*"); + if (re->parse_flags() & Regexp::NonGreedy) + t_->append("?"); + if (prec < PrecUnary) + t_->append(")"); + break; + + case kRegexpPlus: + t_->append("+"); + if (re->parse_flags() & Regexp::NonGreedy) + t_->append("?"); + if (prec < PrecUnary) + t_->append(")"); + break; + + case kRegexpQuest: + t_->append("?"); + if (re->parse_flags() & Regexp::NonGreedy) + t_->append("?"); + if (prec < PrecUnary) + t_->append(")"); + break; + + case kRegexpRepeat: + if (re->max() == -1) + t_->append(StringPrintf("{%d,}", re->min())); + else if (re->min() == re->max()) + t_->append(StringPrintf("{%d}", re->min())); + else + t_->append(StringPrintf("{%d,%d}", re->min(), re->max())); + if (re->parse_flags() & Regexp::NonGreedy) + t_->append("?"); + if (prec < PrecUnary) + t_->append(")"); + break; + + case kRegexpAnyChar: + t_->append("."); + break; + + case kRegexpAnyByte: + t_->append("\\C"); + break; + + case kRegexpBeginLine: + t_->append("^"); + break; + + case kRegexpEndLine: + t_->append("$"); + break; + + case kRegexpBeginText: + t_->append("(?-m:^)"); + break; + + case kRegexpEndText: + if (re->parse_flags() & Regexp::WasDollar) + t_->append("(?-m:$)"); + else + t_->append("\\z"); + break; + + case kRegexpWordBoundary: + t_->append("\\b"); + break; + + case kRegexpNoWordBoundary: + t_->append("\\B"); + break; + + case kRegexpCharClass: { + if (re->cc()->size() == 0) { + t_->append("[^\\x00-\\x{10ffff}]"); + break; + } + t_->append("["); + // Heuristic: show class as negated if it contains the + // non-character 0xFFFE and yet somehow isn't full. + CharClass* cc = re->cc(); + if (cc->Contains(0xFFFE) && !cc->full()) { + cc = cc->Negate(); + t_->append("^"); + } + for (CharClass::iterator i = cc->begin(); i != cc->end(); ++i) + AppendCCRange(t_, i->lo, i->hi); + if (cc != re->cc()) + cc->Delete(); + t_->append("]"); + break; + } + + case kRegexpCapture: + t_->append(")"); + break; + + case kRegexpHaveMatch: + // There's no syntax accepted by the parser to generate + // this node (it is generated by RE2::Set) so make something + // up that is readable but won't compile. + t_->append("(?HaveMatch:%d)", re->match_id()); + break; + } + + // If the parent is an alternation, append the | for it. + if (prec == PrecAlternate) + t_->append("|"); + + return 0; +} + +// Appends a rune for use in a character class to the string t. +static void AppendCCChar(std::string* t, Rune r) { + if (0x20 <= r && r <= 0x7E) { + if (strchr("[]^-\\", r)) + t->append("\\"); + t->append(1, static_cast(r)); + return; + } + switch (r) { + default: + break; + + case '\r': + t->append("\\r"); + return; + + case '\t': + t->append("\\t"); + return; + + case '\n': + t->append("\\n"); + return; + + case '\f': + t->append("\\f"); + return; + } + + if (r < 0x100) { + *t += StringPrintf("\\x%02x", static_cast(r)); + return; + } + *t += StringPrintf("\\x{%x}", static_cast(r)); +} + +static void AppendCCRange(std::string* t, Rune lo, Rune hi) { + if (lo > hi) + return; + AppendCCChar(t, lo); + if (lo < hi) { + t->append("-"); + AppendCCChar(t, hi); + } +} + +} // namespace re2 diff --git a/extern/re2/re2/unicode.py b/extern/re2/re2/unicode.py new file mode 100644 index 0000000000..56ca8119c6 --- /dev/null +++ b/extern/re2/re2/unicode.py @@ -0,0 +1,303 @@ +# Copyright 2008 The RE2 Authors. All Rights Reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +"""Parser for Unicode data files (as distributed by unicode.org).""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import re +from six.moves import urllib + +# Directory or URL where Unicode tables reside. +_UNICODE_DIR = "https://www.unicode.org/Public/12.1.0/ucd" + +# Largest valid Unicode code value. +_RUNE_MAX = 0x10FFFF + + +class Error(Exception): + """Unicode error base class.""" + + +class InputError(Error): + """Unicode input error class. Raised on invalid input.""" + + +def _UInt(s): + """Converts string to Unicode code point ('263A' => 0x263a). + + Args: + s: string to convert + + Returns: + Unicode code point + + Raises: + InputError: the string is not a valid Unicode value. + """ + + try: + v = int(s, 16) + except ValueError: + v = -1 + if len(s) < 4 or len(s) > 6 or v < 0 or v > _RUNE_MAX: + raise InputError("invalid Unicode value %s" % (s,)) + return v + + +def _URange(s): + """Converts string to Unicode range. + + '0001..0003' => [1, 2, 3]. + '0001' => [1]. + + Args: + s: string to convert + + Returns: + Unicode range + + Raises: + InputError: the string is not a valid Unicode range. + """ + a = s.split("..") + if len(a) == 1: + return [_UInt(a[0])] + if len(a) == 2: + lo = _UInt(a[0]) + hi = _UInt(a[1]) + if lo < hi: + return range(lo, hi + 1) + raise InputError("invalid Unicode range %s" % (s,)) + + +def _UStr(v): + """Converts Unicode code point to hex string. + + 0x263a => '0x263A'. + + Args: + v: code point to convert + + Returns: + Unicode string + + Raises: + InputError: the argument is not a valid Unicode value. + """ + if v < 0 or v > _RUNE_MAX: + raise InputError("invalid Unicode value %s" % (v,)) + return "0x%04X" % (v,) + + +def _ParseContinue(s): + """Parses a Unicode continuation field. + + These are of the form '' or ''. + Instead of giving an explicit range in a single table entry, + some Unicode tables use two entries, one for the first + code value in the range and one for the last. + The first entry's description is '' instead of 'Name' + and the second is ''. + + '' => ('Name', 'First') + '' => ('Name', 'Last') + 'Anything else' => ('Anything else', None) + + Args: + s: continuation field string + + Returns: + pair: name and ('First', 'Last', or None) + """ + + match = re.match("<(.*), (First|Last)>", s) + if match is not None: + return match.groups() + return (s, None) + + +def ReadUnicodeTable(filename, nfields, doline): + """Generic Unicode table text file reader. + + The reader takes care of stripping out comments and also + parsing the two different ways that the Unicode tables specify + code ranges (using the .. notation and splitting the range across + multiple lines). + + Each non-comment line in the table is expected to have the given + number of fields. The first field is known to be the Unicode value + and the second field its description. + + The reader calls doline(codes, fields) for each entry in the table. + If fn raises an exception, the reader prints that exception, + prefixed with the file name and line number, and continues + processing the file. When done with the file, the reader re-raises + the first exception encountered during the file. + + Arguments: + filename: the Unicode data file to read, or a file-like object. + nfields: the number of expected fields per line in that file. + doline: the function to call for each table entry. + + Raises: + InputError: nfields is invalid (must be >= 2). + """ + + if nfields < 2: + raise InputError("invalid number of fields %d" % (nfields,)) + + if type(filename) == str: + if filename.startswith("https://"): + fil = urllib.request.urlopen(filename) + else: + fil = open(filename, "rb") + else: + fil = filename + + first = None # first code in multiline range + expect_last = None # tag expected for "Last" line in multiline range + lineno = 0 # current line number + for line in fil: + lineno += 1 + try: + line = line.decode('latin1') + + # Chop # comments and white space; ignore empty lines. + sharp = line.find("#") + if sharp >= 0: + line = line[:sharp] + line = line.strip() + if not line: + continue + + # Split fields on ";", chop more white space. + # Must have the expected number of fields. + fields = [s.strip() for s in line.split(";")] + if len(fields) != nfields: + raise InputError("wrong number of fields %d %d - %s" % + (len(fields), nfields, line)) + + # The Unicode text files have two different ways + # to list a Unicode range. Either the first field is + # itself a range (0000..FFFF), or the range is split + # across two lines, with the second field noting + # the continuation. + codes = _URange(fields[0]) + (name, cont) = _ParseContinue(fields[1]) + + if expect_last is not None: + # If the last line gave the First code in a range, + # this one had better give the Last one. + if (len(codes) != 1 or codes[0] <= first or + cont != "Last" or name != expect_last): + raise InputError("expected Last line for %s" % + (expect_last,)) + codes = range(first, codes[0] + 1) + first = None + expect_last = None + fields[0] = "%04X..%04X" % (codes[0], codes[-1]) + fields[1] = name + elif cont == "First": + # Otherwise, if this is the First code in a range, + # remember it and go to the next line. + if len(codes) != 1: + raise InputError("bad First line: range given") + expect_last = name + first = codes[0] + continue + + doline(codes, fields) + + except Exception as e: + print("%s:%d: %s" % (filename, lineno, e)) + raise + + if expect_last is not None: + raise InputError("expected Last line for %s; got EOF" % + (expect_last,)) + + +def CaseGroups(unicode_dir=_UNICODE_DIR): + """Returns list of Unicode code groups equivalent under case folding. + + Each group is a sorted list of code points, + and the list of groups is sorted by first code point + in the group. + + Args: + unicode_dir: Unicode data directory + + Returns: + list of Unicode code groups + """ + + # Dict mapping lowercase code point to fold-equivalent group. + togroup = {} + + def DoLine(codes, fields): + """Process single CaseFolding.txt line, updating togroup.""" + (_, foldtype, lower, _) = fields + if foldtype not in ("C", "S"): + return + lower = _UInt(lower) + togroup.setdefault(lower, [lower]).extend(codes) + + ReadUnicodeTable(unicode_dir+"/CaseFolding.txt", 4, DoLine) + + groups = list(togroup.values()) + for g in groups: + g.sort() + groups.sort() + return togroup, groups + + +def Scripts(unicode_dir=_UNICODE_DIR): + """Returns dict mapping script names to code lists. + + Args: + unicode_dir: Unicode data directory + + Returns: + dict mapping script names to code lists + """ + + scripts = {} + + def DoLine(codes, fields): + """Process single Scripts.txt line, updating scripts.""" + (_, name) = fields + scripts.setdefault(name, []).extend(codes) + + ReadUnicodeTable(unicode_dir+"/Scripts.txt", 2, DoLine) + return scripts + + +def Categories(unicode_dir=_UNICODE_DIR): + """Returns dict mapping category names to code lists. + + Args: + unicode_dir: Unicode data directory + + Returns: + dict mapping category names to code lists + """ + + categories = {} + + def DoLine(codes, fields): + """Process single UnicodeData.txt line, updating categories.""" + category = fields[2] + categories.setdefault(category, []).extend(codes) + # Add codes from Lu into L, etc. + if len(category) > 1: + short = category[0] + categories.setdefault(short, []).extend(codes) + + ReadUnicodeTable(unicode_dir+"/UnicodeData.txt", 15, DoLine) + return categories + diff --git a/extern/re2/re2/unicode_casefold.cc b/extern/re2/re2/unicode_casefold.cc new file mode 100644 index 0000000000..4ea2533eda --- /dev/null +++ b/extern/re2/re2/unicode_casefold.cc @@ -0,0 +1,578 @@ + +// GENERATED BY make_unicode_casefold.py; DO NOT EDIT. +// make_unicode_casefold.py >unicode_casefold.cc + +#include "re2/unicode_casefold.h" + +namespace re2 { + + +// 1381 groups, 2792 pairs, 356 ranges +const CaseFold unicode_casefold[] = { + { 65, 90, 32 }, + { 97, 106, -32 }, + { 107, 107, 8383 }, + { 108, 114, -32 }, + { 115, 115, 268 }, + { 116, 122, -32 }, + { 181, 181, 743 }, + { 192, 214, 32 }, + { 216, 222, 32 }, + { 223, 223, 7615 }, + { 224, 228, -32 }, + { 229, 229, 8262 }, + { 230, 246, -32 }, + { 248, 254, -32 }, + { 255, 255, 121 }, + { 256, 303, EvenOdd }, + { 306, 311, EvenOdd }, + { 313, 328, OddEven }, + { 330, 375, EvenOdd }, + { 376, 376, -121 }, + { 377, 382, OddEven }, + { 383, 383, -300 }, + { 384, 384, 195 }, + { 385, 385, 210 }, + { 386, 389, EvenOdd }, + { 390, 390, 206 }, + { 391, 392, OddEven }, + { 393, 394, 205 }, + { 395, 396, OddEven }, + { 398, 398, 79 }, + { 399, 399, 202 }, + { 400, 400, 203 }, + { 401, 402, OddEven }, + { 403, 403, 205 }, + { 404, 404, 207 }, + { 405, 405, 97 }, + { 406, 406, 211 }, + { 407, 407, 209 }, + { 408, 409, EvenOdd }, + { 410, 410, 163 }, + { 412, 412, 211 }, + { 413, 413, 213 }, + { 414, 414, 130 }, + { 415, 415, 214 }, + { 416, 421, EvenOdd }, + { 422, 422, 218 }, + { 423, 424, OddEven }, + { 425, 425, 218 }, + { 428, 429, EvenOdd }, + { 430, 430, 218 }, + { 431, 432, OddEven }, + { 433, 434, 217 }, + { 435, 438, OddEven }, + { 439, 439, 219 }, + { 440, 441, EvenOdd }, + { 444, 445, EvenOdd }, + { 447, 447, 56 }, + { 452, 452, EvenOdd }, + { 453, 453, OddEven }, + { 454, 454, -2 }, + { 455, 455, OddEven }, + { 456, 456, EvenOdd }, + { 457, 457, -2 }, + { 458, 458, EvenOdd }, + { 459, 459, OddEven }, + { 460, 460, -2 }, + { 461, 476, OddEven }, + { 477, 477, -79 }, + { 478, 495, EvenOdd }, + { 497, 497, OddEven }, + { 498, 498, EvenOdd }, + { 499, 499, -2 }, + { 500, 501, EvenOdd }, + { 502, 502, -97 }, + { 503, 503, -56 }, + { 504, 543, EvenOdd }, + { 544, 544, -130 }, + { 546, 563, EvenOdd }, + { 570, 570, 10795 }, + { 571, 572, OddEven }, + { 573, 573, -163 }, + { 574, 574, 10792 }, + { 575, 576, 10815 }, + { 577, 578, OddEven }, + { 579, 579, -195 }, + { 580, 580, 69 }, + { 581, 581, 71 }, + { 582, 591, EvenOdd }, + { 592, 592, 10783 }, + { 593, 593, 10780 }, + { 594, 594, 10782 }, + { 595, 595, -210 }, + { 596, 596, -206 }, + { 598, 599, -205 }, + { 601, 601, -202 }, + { 603, 603, -203 }, + { 604, 604, 42319 }, + { 608, 608, -205 }, + { 609, 609, 42315 }, + { 611, 611, -207 }, + { 613, 613, 42280 }, + { 614, 614, 42308 }, + { 616, 616, -209 }, + { 617, 617, -211 }, + { 618, 618, 42308 }, + { 619, 619, 10743 }, + { 620, 620, 42305 }, + { 623, 623, -211 }, + { 625, 625, 10749 }, + { 626, 626, -213 }, + { 629, 629, -214 }, + { 637, 637, 10727 }, + { 640, 640, -218 }, + { 642, 642, 42307 }, + { 643, 643, -218 }, + { 647, 647, 42282 }, + { 648, 648, -218 }, + { 649, 649, -69 }, + { 650, 651, -217 }, + { 652, 652, -71 }, + { 658, 658, -219 }, + { 669, 669, 42261 }, + { 670, 670, 42258 }, + { 837, 837, 84 }, + { 880, 883, EvenOdd }, + { 886, 887, EvenOdd }, + { 891, 893, 130 }, + { 895, 895, 116 }, + { 902, 902, 38 }, + { 904, 906, 37 }, + { 908, 908, 64 }, + { 910, 911, 63 }, + { 913, 929, 32 }, + { 931, 931, 31 }, + { 932, 939, 32 }, + { 940, 940, -38 }, + { 941, 943, -37 }, + { 945, 945, -32 }, + { 946, 946, 30 }, + { 947, 948, -32 }, + { 949, 949, 64 }, + { 950, 951, -32 }, + { 952, 952, 25 }, + { 953, 953, 7173 }, + { 954, 954, 54 }, + { 955, 955, -32 }, + { 956, 956, -775 }, + { 957, 959, -32 }, + { 960, 960, 22 }, + { 961, 961, 48 }, + { 962, 962, EvenOdd }, + { 963, 965, -32 }, + { 966, 966, 15 }, + { 967, 968, -32 }, + { 969, 969, 7517 }, + { 970, 971, -32 }, + { 972, 972, -64 }, + { 973, 974, -63 }, + { 975, 975, 8 }, + { 976, 976, -62 }, + { 977, 977, 35 }, + { 981, 981, -47 }, + { 982, 982, -54 }, + { 983, 983, -8 }, + { 984, 1007, EvenOdd }, + { 1008, 1008, -86 }, + { 1009, 1009, -80 }, + { 1010, 1010, 7 }, + { 1011, 1011, -116 }, + { 1012, 1012, -92 }, + { 1013, 1013, -96 }, + { 1015, 1016, OddEven }, + { 1017, 1017, -7 }, + { 1018, 1019, EvenOdd }, + { 1021, 1023, -130 }, + { 1024, 1039, 80 }, + { 1040, 1071, 32 }, + { 1072, 1073, -32 }, + { 1074, 1074, 6222 }, + { 1075, 1075, -32 }, + { 1076, 1076, 6221 }, + { 1077, 1085, -32 }, + { 1086, 1086, 6212 }, + { 1087, 1088, -32 }, + { 1089, 1090, 6210 }, + { 1091, 1097, -32 }, + { 1098, 1098, 6204 }, + { 1099, 1103, -32 }, + { 1104, 1119, -80 }, + { 1120, 1122, EvenOdd }, + { 1123, 1123, 6180 }, + { 1124, 1153, EvenOdd }, + { 1162, 1215, EvenOdd }, + { 1216, 1216, 15 }, + { 1217, 1230, OddEven }, + { 1231, 1231, -15 }, + { 1232, 1327, EvenOdd }, + { 1329, 1366, 48 }, + { 1377, 1414, -48 }, + { 4256, 4293, 7264 }, + { 4295, 4295, 7264 }, + { 4301, 4301, 7264 }, + { 4304, 4346, 3008 }, + { 4349, 4351, 3008 }, + { 5024, 5103, 38864 }, + { 5104, 5109, 8 }, + { 5112, 5117, -8 }, + { 7296, 7296, -6254 }, + { 7297, 7297, -6253 }, + { 7298, 7298, -6244 }, + { 7299, 7299, -6242 }, + { 7300, 7300, EvenOdd }, + { 7301, 7301, -6243 }, + { 7302, 7302, -6236 }, + { 7303, 7303, -6181 }, + { 7304, 7304, 35266 }, + { 7312, 7354, -3008 }, + { 7357, 7359, -3008 }, + { 7545, 7545, 35332 }, + { 7549, 7549, 3814 }, + { 7566, 7566, 35384 }, + { 7680, 7776, EvenOdd }, + { 7777, 7777, 58 }, + { 7778, 7829, EvenOdd }, + { 7835, 7835, -59 }, + { 7838, 7838, -7615 }, + { 7840, 7935, EvenOdd }, + { 7936, 7943, 8 }, + { 7944, 7951, -8 }, + { 7952, 7957, 8 }, + { 7960, 7965, -8 }, + { 7968, 7975, 8 }, + { 7976, 7983, -8 }, + { 7984, 7991, 8 }, + { 7992, 7999, -8 }, + { 8000, 8005, 8 }, + { 8008, 8013, -8 }, + { 8017, 8017, 8 }, + { 8019, 8019, 8 }, + { 8021, 8021, 8 }, + { 8023, 8023, 8 }, + { 8025, 8025, -8 }, + { 8027, 8027, -8 }, + { 8029, 8029, -8 }, + { 8031, 8031, -8 }, + { 8032, 8039, 8 }, + { 8040, 8047, -8 }, + { 8048, 8049, 74 }, + { 8050, 8053, 86 }, + { 8054, 8055, 100 }, + { 8056, 8057, 128 }, + { 8058, 8059, 112 }, + { 8060, 8061, 126 }, + { 8064, 8071, 8 }, + { 8072, 8079, -8 }, + { 8080, 8087, 8 }, + { 8088, 8095, -8 }, + { 8096, 8103, 8 }, + { 8104, 8111, -8 }, + { 8112, 8113, 8 }, + { 8115, 8115, 9 }, + { 8120, 8121, -8 }, + { 8122, 8123, -74 }, + { 8124, 8124, -9 }, + { 8126, 8126, -7289 }, + { 8131, 8131, 9 }, + { 8136, 8139, -86 }, + { 8140, 8140, -9 }, + { 8144, 8145, 8 }, + { 8152, 8153, -8 }, + { 8154, 8155, -100 }, + { 8160, 8161, 8 }, + { 8165, 8165, 7 }, + { 8168, 8169, -8 }, + { 8170, 8171, -112 }, + { 8172, 8172, -7 }, + { 8179, 8179, 9 }, + { 8184, 8185, -128 }, + { 8186, 8187, -126 }, + { 8188, 8188, -9 }, + { 8486, 8486, -7549 }, + { 8490, 8490, -8415 }, + { 8491, 8491, -8294 }, + { 8498, 8498, 28 }, + { 8526, 8526, -28 }, + { 8544, 8559, 16 }, + { 8560, 8575, -16 }, + { 8579, 8580, OddEven }, + { 9398, 9423, 26 }, + { 9424, 9449, -26 }, + { 11264, 11310, 48 }, + { 11312, 11358, -48 }, + { 11360, 11361, EvenOdd }, + { 11362, 11362, -10743 }, + { 11363, 11363, -3814 }, + { 11364, 11364, -10727 }, + { 11365, 11365, -10795 }, + { 11366, 11366, -10792 }, + { 11367, 11372, OddEven }, + { 11373, 11373, -10780 }, + { 11374, 11374, -10749 }, + { 11375, 11375, -10783 }, + { 11376, 11376, -10782 }, + { 11378, 11379, EvenOdd }, + { 11381, 11382, OddEven }, + { 11390, 11391, -10815 }, + { 11392, 11491, EvenOdd }, + { 11499, 11502, OddEven }, + { 11506, 11507, EvenOdd }, + { 11520, 11557, -7264 }, + { 11559, 11559, -7264 }, + { 11565, 11565, -7264 }, + { 42560, 42570, EvenOdd }, + { 42571, 42571, -35267 }, + { 42572, 42605, EvenOdd }, + { 42624, 42651, EvenOdd }, + { 42786, 42799, EvenOdd }, + { 42802, 42863, EvenOdd }, + { 42873, 42876, OddEven }, + { 42877, 42877, -35332 }, + { 42878, 42887, EvenOdd }, + { 42891, 42892, OddEven }, + { 42893, 42893, -42280 }, + { 42896, 42899, EvenOdd }, + { 42900, 42900, 48 }, + { 42902, 42921, EvenOdd }, + { 42922, 42922, -42308 }, + { 42923, 42923, -42319 }, + { 42924, 42924, -42315 }, + { 42925, 42925, -42305 }, + { 42926, 42926, -42308 }, + { 42928, 42928, -42258 }, + { 42929, 42929, -42282 }, + { 42930, 42930, -42261 }, + { 42931, 42931, 928 }, + { 42932, 42943, EvenOdd }, + { 42946, 42947, EvenOdd }, + { 42948, 42948, -48 }, + { 42949, 42949, -42307 }, + { 42950, 42950, -35384 }, + { 43859, 43859, -928 }, + { 43888, 43967, -38864 }, + { 65313, 65338, 32 }, + { 65345, 65370, -32 }, + { 66560, 66599, 40 }, + { 66600, 66639, -40 }, + { 66736, 66771, 40 }, + { 66776, 66811, -40 }, + { 68736, 68786, 64 }, + { 68800, 68850, -64 }, + { 71840, 71871, 32 }, + { 71872, 71903, -32 }, + { 93760, 93791, 32 }, + { 93792, 93823, -32 }, + { 125184, 125217, 34 }, + { 125218, 125251, -34 }, +}; +const int num_unicode_casefold = 356; + +// 1381 groups, 1411 pairs, 198 ranges +const CaseFold unicode_tolower[] = { + { 65, 90, 32 }, + { 181, 181, 775 }, + { 192, 214, 32 }, + { 216, 222, 32 }, + { 256, 302, EvenOddSkip }, + { 306, 310, EvenOddSkip }, + { 313, 327, OddEvenSkip }, + { 330, 374, EvenOddSkip }, + { 376, 376, -121 }, + { 377, 381, OddEvenSkip }, + { 383, 383, -268 }, + { 385, 385, 210 }, + { 386, 388, EvenOddSkip }, + { 390, 390, 206 }, + { 391, 391, OddEven }, + { 393, 394, 205 }, + { 395, 395, OddEven }, + { 398, 398, 79 }, + { 399, 399, 202 }, + { 400, 400, 203 }, + { 401, 401, OddEven }, + { 403, 403, 205 }, + { 404, 404, 207 }, + { 406, 406, 211 }, + { 407, 407, 209 }, + { 408, 408, EvenOdd }, + { 412, 412, 211 }, + { 413, 413, 213 }, + { 415, 415, 214 }, + { 416, 420, EvenOddSkip }, + { 422, 422, 218 }, + { 423, 423, OddEven }, + { 425, 425, 218 }, + { 428, 428, EvenOdd }, + { 430, 430, 218 }, + { 431, 431, OddEven }, + { 433, 434, 217 }, + { 435, 437, OddEvenSkip }, + { 439, 439, 219 }, + { 440, 440, EvenOdd }, + { 444, 444, EvenOdd }, + { 452, 452, 2 }, + { 453, 453, OddEven }, + { 455, 455, 2 }, + { 456, 456, EvenOdd }, + { 458, 458, 2 }, + { 459, 475, OddEvenSkip }, + { 478, 494, EvenOddSkip }, + { 497, 497, 2 }, + { 498, 500, EvenOddSkip }, + { 502, 502, -97 }, + { 503, 503, -56 }, + { 504, 542, EvenOddSkip }, + { 544, 544, -130 }, + { 546, 562, EvenOddSkip }, + { 570, 570, 10795 }, + { 571, 571, OddEven }, + { 573, 573, -163 }, + { 574, 574, 10792 }, + { 577, 577, OddEven }, + { 579, 579, -195 }, + { 580, 580, 69 }, + { 581, 581, 71 }, + { 582, 590, EvenOddSkip }, + { 837, 837, 116 }, + { 880, 882, EvenOddSkip }, + { 886, 886, EvenOdd }, + { 895, 895, 116 }, + { 902, 902, 38 }, + { 904, 906, 37 }, + { 908, 908, 64 }, + { 910, 911, 63 }, + { 913, 929, 32 }, + { 931, 939, 32 }, + { 962, 962, EvenOdd }, + { 975, 975, 8 }, + { 976, 976, -30 }, + { 977, 977, -25 }, + { 981, 981, -15 }, + { 982, 982, -22 }, + { 984, 1006, EvenOddSkip }, + { 1008, 1008, -54 }, + { 1009, 1009, -48 }, + { 1012, 1012, -60 }, + { 1013, 1013, -64 }, + { 1015, 1015, OddEven }, + { 1017, 1017, -7 }, + { 1018, 1018, EvenOdd }, + { 1021, 1023, -130 }, + { 1024, 1039, 80 }, + { 1040, 1071, 32 }, + { 1120, 1152, EvenOddSkip }, + { 1162, 1214, EvenOddSkip }, + { 1216, 1216, 15 }, + { 1217, 1229, OddEvenSkip }, + { 1232, 1326, EvenOddSkip }, + { 1329, 1366, 48 }, + { 4256, 4293, 7264 }, + { 4295, 4295, 7264 }, + { 4301, 4301, 7264 }, + { 5112, 5117, -8 }, + { 7296, 7296, -6222 }, + { 7297, 7297, -6221 }, + { 7298, 7298, -6212 }, + { 7299, 7300, -6210 }, + { 7301, 7301, -6211 }, + { 7302, 7302, -6204 }, + { 7303, 7303, -6180 }, + { 7304, 7304, 35267 }, + { 7312, 7354, -3008 }, + { 7357, 7359, -3008 }, + { 7680, 7828, EvenOddSkip }, + { 7835, 7835, -58 }, + { 7838, 7838, -7615 }, + { 7840, 7934, EvenOddSkip }, + { 7944, 7951, -8 }, + { 7960, 7965, -8 }, + { 7976, 7983, -8 }, + { 7992, 7999, -8 }, + { 8008, 8013, -8 }, + { 8025, 8025, -8 }, + { 8027, 8027, -8 }, + { 8029, 8029, -8 }, + { 8031, 8031, -8 }, + { 8040, 8047, -8 }, + { 8072, 8079, -8 }, + { 8088, 8095, -8 }, + { 8104, 8111, -8 }, + { 8120, 8121, -8 }, + { 8122, 8123, -74 }, + { 8124, 8124, -9 }, + { 8126, 8126, -7173 }, + { 8136, 8139, -86 }, + { 8140, 8140, -9 }, + { 8152, 8153, -8 }, + { 8154, 8155, -100 }, + { 8168, 8169, -8 }, + { 8170, 8171, -112 }, + { 8172, 8172, -7 }, + { 8184, 8185, -128 }, + { 8186, 8187, -126 }, + { 8188, 8188, -9 }, + { 8486, 8486, -7517 }, + { 8490, 8490, -8383 }, + { 8491, 8491, -8262 }, + { 8498, 8498, 28 }, + { 8544, 8559, 16 }, + { 8579, 8579, OddEven }, + { 9398, 9423, 26 }, + { 11264, 11310, 48 }, + { 11360, 11360, EvenOdd }, + { 11362, 11362, -10743 }, + { 11363, 11363, -3814 }, + { 11364, 11364, -10727 }, + { 11367, 11371, OddEvenSkip }, + { 11373, 11373, -10780 }, + { 11374, 11374, -10749 }, + { 11375, 11375, -10783 }, + { 11376, 11376, -10782 }, + { 11378, 11378, EvenOdd }, + { 11381, 11381, OddEven }, + { 11390, 11391, -10815 }, + { 11392, 11490, EvenOddSkip }, + { 11499, 11501, OddEvenSkip }, + { 11506, 11506, EvenOdd }, + { 42560, 42604, EvenOddSkip }, + { 42624, 42650, EvenOddSkip }, + { 42786, 42798, EvenOddSkip }, + { 42802, 42862, EvenOddSkip }, + { 42873, 42875, OddEvenSkip }, + { 42877, 42877, -35332 }, + { 42878, 42886, EvenOddSkip }, + { 42891, 42891, OddEven }, + { 42893, 42893, -42280 }, + { 42896, 42898, EvenOddSkip }, + { 42902, 42920, EvenOddSkip }, + { 42922, 42922, -42308 }, + { 42923, 42923, -42319 }, + { 42924, 42924, -42315 }, + { 42925, 42925, -42305 }, + { 42926, 42926, -42308 }, + { 42928, 42928, -42258 }, + { 42929, 42929, -42282 }, + { 42930, 42930, -42261 }, + { 42931, 42931, 928 }, + { 42932, 42942, EvenOddSkip }, + { 42946, 42946, EvenOdd }, + { 42948, 42948, -48 }, + { 42949, 42949, -42307 }, + { 42950, 42950, -35384 }, + { 43888, 43967, -38864 }, + { 65313, 65338, 32 }, + { 66560, 66599, 40 }, + { 66736, 66771, 40 }, + { 68736, 68786, 64 }, + { 71840, 71871, 32 }, + { 93760, 93791, 32 }, + { 125184, 125217, 34 }, +}; +const int num_unicode_tolower = 198; + + + +} // namespace re2 + + diff --git a/extern/re2/re2/unicode_casefold.h b/extern/re2/re2/unicode_casefold.h new file mode 100644 index 0000000000..8bdbb42fbc --- /dev/null +++ b/extern/re2/re2/unicode_casefold.h @@ -0,0 +1,78 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_UNICODE_CASEFOLD_H_ +#define RE2_UNICODE_CASEFOLD_H_ + +// Unicode case folding tables. + +// The Unicode case folding tables encode the mapping from one Unicode point +// to the next largest Unicode point with equivalent folding. The largest +// point wraps back to the first. For example, the tables map: +// +// 'A' -> 'a' +// 'a' -> 'A' +// +// 'K' -> 'k' +// 'k' -> 'K' (Kelvin symbol) +// 'K' -> 'K' +// +// Like everything Unicode, these tables are big. If we represent the table +// as a sorted list of uint32_t pairs, it has 2049 entries and is 16 kB. +// Most table entries look like the ones around them: +// 'A' maps to 'A'+32, 'B' maps to 'B'+32, etc. +// Instead of listing all the pairs explicitly, we make a list of ranges +// and deltas, so that the table entries for 'A' through 'Z' can be represented +// as a single entry { 'A', 'Z', +32 }. +// +// In addition to blocks that map to each other (A-Z mapping to a-z) +// there are blocks of pairs that individually map to each other +// (for example, 0100<->0101, 0102<->0103, 0104<->0105, ...). +// For those, the special delta value EvenOdd marks even/odd pairs +// (if even, add 1; if odd, subtract 1), and OddEven marks odd/even pairs. +// +// In this form, the table has 274 entries, about 3kB. If we were to split +// the table into one for 16-bit codes and an overflow table for larger ones, +// we could get it down to about 1.5kB, but that's not worth the complexity. +// +// The grouped form also allows for efficient fold range calculations +// rather than looping one character at a time. + +#include + +#include "util/util.h" +#include "util/utf.h" + +namespace re2 { + +enum { + EvenOdd = 1, + OddEven = -1, + EvenOddSkip = 1<<30, + OddEvenSkip, +}; + +struct CaseFold { + Rune lo; + Rune hi; + int32_t delta; +}; + +extern const CaseFold unicode_casefold[]; +extern const int num_unicode_casefold; + +extern const CaseFold unicode_tolower[]; +extern const int num_unicode_tolower; + +// Returns the CaseFold* in the tables that contains rune. +// If rune is not in the tables, returns the first CaseFold* after rune. +// If rune is larger than any value in the tables, returns NULL. +extern const CaseFold* LookupCaseFold(const CaseFold*, int, Rune rune); + +// Returns the result of applying the fold f to the rune r. +extern Rune ApplyFold(const CaseFold *f, Rune r); + +} // namespace re2 + +#endif // RE2_UNICODE_CASEFOLD_H_ diff --git a/extern/re2/re2/unicode_groups.cc b/extern/re2/re2/unicode_groups.cc new file mode 100644 index 0000000000..63e611658c --- /dev/null +++ b/extern/re2/re2/unicode_groups.cc @@ -0,0 +1,6162 @@ + +// GENERATED BY make_unicode_groups.py; DO NOT EDIT. +// make_unicode_groups.py >unicode_groups.cc + +#include "re2/unicode_groups.h" + +namespace re2 { + + +static const URange16 C_range16[] = { + { 0, 31 }, + { 127, 159 }, + { 173, 173 }, + { 1536, 1541 }, + { 1564, 1564 }, + { 1757, 1757 }, + { 1807, 1807 }, + { 2274, 2274 }, + { 6158, 6158 }, + { 8203, 8207 }, + { 8234, 8238 }, + { 8288, 8292 }, + { 8294, 8303 }, + { 55296, 63743 }, + { 65279, 65279 }, + { 65529, 65531 }, +}; +static const URange32 C_range32[] = { + { 69821, 69821 }, + { 69837, 69837 }, + { 78896, 78904 }, + { 113824, 113827 }, + { 119155, 119162 }, + { 917505, 917505 }, + { 917536, 917631 }, + { 983040, 1048573 }, + { 1048576, 1114109 }, +}; +static const URange16 Cc_range16[] = { + { 0, 31 }, + { 127, 159 }, +}; +static const URange16 Cf_range16[] = { + { 173, 173 }, + { 1536, 1541 }, + { 1564, 1564 }, + { 1757, 1757 }, + { 1807, 1807 }, + { 2274, 2274 }, + { 6158, 6158 }, + { 8203, 8207 }, + { 8234, 8238 }, + { 8288, 8292 }, + { 8294, 8303 }, + { 65279, 65279 }, + { 65529, 65531 }, +}; +static const URange32 Cf_range32[] = { + { 69821, 69821 }, + { 69837, 69837 }, + { 78896, 78904 }, + { 113824, 113827 }, + { 119155, 119162 }, + { 917505, 917505 }, + { 917536, 917631 }, +}; +static const URange16 Co_range16[] = { + { 57344, 63743 }, +}; +static const URange32 Co_range32[] = { + { 983040, 1048573 }, + { 1048576, 1114109 }, +}; +static const URange16 Cs_range16[] = { + { 55296, 57343 }, +}; +static const URange16 L_range16[] = { + { 65, 90 }, + { 97, 122 }, + { 170, 170 }, + { 181, 181 }, + { 186, 186 }, + { 192, 214 }, + { 216, 246 }, + { 248, 705 }, + { 710, 721 }, + { 736, 740 }, + { 748, 748 }, + { 750, 750 }, + { 880, 884 }, + { 886, 887 }, + { 890, 893 }, + { 895, 895 }, + { 902, 902 }, + { 904, 906 }, + { 908, 908 }, + { 910, 929 }, + { 931, 1013 }, + { 1015, 1153 }, + { 1162, 1327 }, + { 1329, 1366 }, + { 1369, 1369 }, + { 1376, 1416 }, + { 1488, 1514 }, + { 1519, 1522 }, + { 1568, 1610 }, + { 1646, 1647 }, + { 1649, 1747 }, + { 1749, 1749 }, + { 1765, 1766 }, + { 1774, 1775 }, + { 1786, 1788 }, + { 1791, 1791 }, + { 1808, 1808 }, + { 1810, 1839 }, + { 1869, 1957 }, + { 1969, 1969 }, + { 1994, 2026 }, + { 2036, 2037 }, + { 2042, 2042 }, + { 2048, 2069 }, + { 2074, 2074 }, + { 2084, 2084 }, + { 2088, 2088 }, + { 2112, 2136 }, + { 2144, 2154 }, + { 2208, 2228 }, + { 2230, 2237 }, + { 2308, 2361 }, + { 2365, 2365 }, + { 2384, 2384 }, + { 2392, 2401 }, + { 2417, 2432 }, + { 2437, 2444 }, + { 2447, 2448 }, + { 2451, 2472 }, + { 2474, 2480 }, + { 2482, 2482 }, + { 2486, 2489 }, + { 2493, 2493 }, + { 2510, 2510 }, + { 2524, 2525 }, + { 2527, 2529 }, + { 2544, 2545 }, + { 2556, 2556 }, + { 2565, 2570 }, + { 2575, 2576 }, + { 2579, 2600 }, + { 2602, 2608 }, + { 2610, 2611 }, + { 2613, 2614 }, + { 2616, 2617 }, + { 2649, 2652 }, + { 2654, 2654 }, + { 2674, 2676 }, + { 2693, 2701 }, + { 2703, 2705 }, + { 2707, 2728 }, + { 2730, 2736 }, + { 2738, 2739 }, + { 2741, 2745 }, + { 2749, 2749 }, + { 2768, 2768 }, + { 2784, 2785 }, + { 2809, 2809 }, + { 2821, 2828 }, + { 2831, 2832 }, + { 2835, 2856 }, + { 2858, 2864 }, + { 2866, 2867 }, + { 2869, 2873 }, + { 2877, 2877 }, + { 2908, 2909 }, + { 2911, 2913 }, + { 2929, 2929 }, + { 2947, 2947 }, + { 2949, 2954 }, + { 2958, 2960 }, + { 2962, 2965 }, + { 2969, 2970 }, + { 2972, 2972 }, + { 2974, 2975 }, + { 2979, 2980 }, + { 2984, 2986 }, + { 2990, 3001 }, + { 3024, 3024 }, + { 3077, 3084 }, + { 3086, 3088 }, + { 3090, 3112 }, + { 3114, 3129 }, + { 3133, 3133 }, + { 3160, 3162 }, + { 3168, 3169 }, + { 3200, 3200 }, + { 3205, 3212 }, + { 3214, 3216 }, + { 3218, 3240 }, + { 3242, 3251 }, + { 3253, 3257 }, + { 3261, 3261 }, + { 3294, 3294 }, + { 3296, 3297 }, + { 3313, 3314 }, + { 3333, 3340 }, + { 3342, 3344 }, + { 3346, 3386 }, + { 3389, 3389 }, + { 3406, 3406 }, + { 3412, 3414 }, + { 3423, 3425 }, + { 3450, 3455 }, + { 3461, 3478 }, + { 3482, 3505 }, + { 3507, 3515 }, + { 3517, 3517 }, + { 3520, 3526 }, + { 3585, 3632 }, + { 3634, 3635 }, + { 3648, 3654 }, + { 3713, 3714 }, + { 3716, 3716 }, + { 3718, 3722 }, + { 3724, 3747 }, + { 3749, 3749 }, + { 3751, 3760 }, + { 3762, 3763 }, + { 3773, 3773 }, + { 3776, 3780 }, + { 3782, 3782 }, + { 3804, 3807 }, + { 3840, 3840 }, + { 3904, 3911 }, + { 3913, 3948 }, + { 3976, 3980 }, + { 4096, 4138 }, + { 4159, 4159 }, + { 4176, 4181 }, + { 4186, 4189 }, + { 4193, 4193 }, + { 4197, 4198 }, + { 4206, 4208 }, + { 4213, 4225 }, + { 4238, 4238 }, + { 4256, 4293 }, + { 4295, 4295 }, + { 4301, 4301 }, + { 4304, 4346 }, + { 4348, 4680 }, + { 4682, 4685 }, + { 4688, 4694 }, + { 4696, 4696 }, + { 4698, 4701 }, + { 4704, 4744 }, + { 4746, 4749 }, + { 4752, 4784 }, + { 4786, 4789 }, + { 4792, 4798 }, + { 4800, 4800 }, + { 4802, 4805 }, + { 4808, 4822 }, + { 4824, 4880 }, + { 4882, 4885 }, + { 4888, 4954 }, + { 4992, 5007 }, + { 5024, 5109 }, + { 5112, 5117 }, + { 5121, 5740 }, + { 5743, 5759 }, + { 5761, 5786 }, + { 5792, 5866 }, + { 5873, 5880 }, + { 5888, 5900 }, + { 5902, 5905 }, + { 5920, 5937 }, + { 5952, 5969 }, + { 5984, 5996 }, + { 5998, 6000 }, + { 6016, 6067 }, + { 6103, 6103 }, + { 6108, 6108 }, + { 6176, 6264 }, + { 6272, 6276 }, + { 6279, 6312 }, + { 6314, 6314 }, + { 6320, 6389 }, + { 6400, 6430 }, + { 6480, 6509 }, + { 6512, 6516 }, + { 6528, 6571 }, + { 6576, 6601 }, + { 6656, 6678 }, + { 6688, 6740 }, + { 6823, 6823 }, + { 6917, 6963 }, + { 6981, 6987 }, + { 7043, 7072 }, + { 7086, 7087 }, + { 7098, 7141 }, + { 7168, 7203 }, + { 7245, 7247 }, + { 7258, 7293 }, + { 7296, 7304 }, + { 7312, 7354 }, + { 7357, 7359 }, + { 7401, 7404 }, + { 7406, 7411 }, + { 7413, 7414 }, + { 7418, 7418 }, + { 7424, 7615 }, + { 7680, 7957 }, + { 7960, 7965 }, + { 7968, 8005 }, + { 8008, 8013 }, + { 8016, 8023 }, + { 8025, 8025 }, + { 8027, 8027 }, + { 8029, 8029 }, + { 8031, 8061 }, + { 8064, 8116 }, + { 8118, 8124 }, + { 8126, 8126 }, + { 8130, 8132 }, + { 8134, 8140 }, + { 8144, 8147 }, + { 8150, 8155 }, + { 8160, 8172 }, + { 8178, 8180 }, + { 8182, 8188 }, + { 8305, 8305 }, + { 8319, 8319 }, + { 8336, 8348 }, + { 8450, 8450 }, + { 8455, 8455 }, + { 8458, 8467 }, + { 8469, 8469 }, + { 8473, 8477 }, + { 8484, 8484 }, + { 8486, 8486 }, + { 8488, 8488 }, + { 8490, 8493 }, + { 8495, 8505 }, + { 8508, 8511 }, + { 8517, 8521 }, + { 8526, 8526 }, + { 8579, 8580 }, + { 11264, 11310 }, + { 11312, 11358 }, + { 11360, 11492 }, + { 11499, 11502 }, + { 11506, 11507 }, + { 11520, 11557 }, + { 11559, 11559 }, + { 11565, 11565 }, + { 11568, 11623 }, + { 11631, 11631 }, + { 11648, 11670 }, + { 11680, 11686 }, + { 11688, 11694 }, + { 11696, 11702 }, + { 11704, 11710 }, + { 11712, 11718 }, + { 11720, 11726 }, + { 11728, 11734 }, + { 11736, 11742 }, + { 11823, 11823 }, + { 12293, 12294 }, + { 12337, 12341 }, + { 12347, 12348 }, + { 12353, 12438 }, + { 12445, 12447 }, + { 12449, 12538 }, + { 12540, 12543 }, + { 12549, 12591 }, + { 12593, 12686 }, + { 12704, 12730 }, + { 12784, 12799 }, + { 13312, 19893 }, + { 19968, 40943 }, + { 40960, 42124 }, + { 42192, 42237 }, + { 42240, 42508 }, + { 42512, 42527 }, + { 42538, 42539 }, + { 42560, 42606 }, + { 42623, 42653 }, + { 42656, 42725 }, + { 42775, 42783 }, + { 42786, 42888 }, + { 42891, 42943 }, + { 42946, 42950 }, + { 42999, 43009 }, + { 43011, 43013 }, + { 43015, 43018 }, + { 43020, 43042 }, + { 43072, 43123 }, + { 43138, 43187 }, + { 43250, 43255 }, + { 43259, 43259 }, + { 43261, 43262 }, + { 43274, 43301 }, + { 43312, 43334 }, + { 43360, 43388 }, + { 43396, 43442 }, + { 43471, 43471 }, + { 43488, 43492 }, + { 43494, 43503 }, + { 43514, 43518 }, + { 43520, 43560 }, + { 43584, 43586 }, + { 43588, 43595 }, + { 43616, 43638 }, + { 43642, 43642 }, + { 43646, 43695 }, + { 43697, 43697 }, + { 43701, 43702 }, + { 43705, 43709 }, + { 43712, 43712 }, + { 43714, 43714 }, + { 43739, 43741 }, + { 43744, 43754 }, + { 43762, 43764 }, + { 43777, 43782 }, + { 43785, 43790 }, + { 43793, 43798 }, + { 43808, 43814 }, + { 43816, 43822 }, + { 43824, 43866 }, + { 43868, 43879 }, + { 43888, 44002 }, + { 44032, 55203 }, + { 55216, 55238 }, + { 55243, 55291 }, + { 63744, 64109 }, + { 64112, 64217 }, + { 64256, 64262 }, + { 64275, 64279 }, + { 64285, 64285 }, + { 64287, 64296 }, + { 64298, 64310 }, + { 64312, 64316 }, + { 64318, 64318 }, + { 64320, 64321 }, + { 64323, 64324 }, + { 64326, 64433 }, + { 64467, 64829 }, + { 64848, 64911 }, + { 64914, 64967 }, + { 65008, 65019 }, + { 65136, 65140 }, + { 65142, 65276 }, + { 65313, 65338 }, + { 65345, 65370 }, + { 65382, 65470 }, + { 65474, 65479 }, + { 65482, 65487 }, + { 65490, 65495 }, + { 65498, 65500 }, +}; +static const URange32 L_range32[] = { + { 65536, 65547 }, + { 65549, 65574 }, + { 65576, 65594 }, + { 65596, 65597 }, + { 65599, 65613 }, + { 65616, 65629 }, + { 65664, 65786 }, + { 66176, 66204 }, + { 66208, 66256 }, + { 66304, 66335 }, + { 66349, 66368 }, + { 66370, 66377 }, + { 66384, 66421 }, + { 66432, 66461 }, + { 66464, 66499 }, + { 66504, 66511 }, + { 66560, 66717 }, + { 66736, 66771 }, + { 66776, 66811 }, + { 66816, 66855 }, + { 66864, 66915 }, + { 67072, 67382 }, + { 67392, 67413 }, + { 67424, 67431 }, + { 67584, 67589 }, + { 67592, 67592 }, + { 67594, 67637 }, + { 67639, 67640 }, + { 67644, 67644 }, + { 67647, 67669 }, + { 67680, 67702 }, + { 67712, 67742 }, + { 67808, 67826 }, + { 67828, 67829 }, + { 67840, 67861 }, + { 67872, 67897 }, + { 67968, 68023 }, + { 68030, 68031 }, + { 68096, 68096 }, + { 68112, 68115 }, + { 68117, 68119 }, + { 68121, 68149 }, + { 68192, 68220 }, + { 68224, 68252 }, + { 68288, 68295 }, + { 68297, 68324 }, + { 68352, 68405 }, + { 68416, 68437 }, + { 68448, 68466 }, + { 68480, 68497 }, + { 68608, 68680 }, + { 68736, 68786 }, + { 68800, 68850 }, + { 68864, 68899 }, + { 69376, 69404 }, + { 69415, 69415 }, + { 69424, 69445 }, + { 69600, 69622 }, + { 69635, 69687 }, + { 69763, 69807 }, + { 69840, 69864 }, + { 69891, 69926 }, + { 69956, 69956 }, + { 69968, 70002 }, + { 70006, 70006 }, + { 70019, 70066 }, + { 70081, 70084 }, + { 70106, 70106 }, + { 70108, 70108 }, + { 70144, 70161 }, + { 70163, 70187 }, + { 70272, 70278 }, + { 70280, 70280 }, + { 70282, 70285 }, + { 70287, 70301 }, + { 70303, 70312 }, + { 70320, 70366 }, + { 70405, 70412 }, + { 70415, 70416 }, + { 70419, 70440 }, + { 70442, 70448 }, + { 70450, 70451 }, + { 70453, 70457 }, + { 70461, 70461 }, + { 70480, 70480 }, + { 70493, 70497 }, + { 70656, 70708 }, + { 70727, 70730 }, + { 70751, 70751 }, + { 70784, 70831 }, + { 70852, 70853 }, + { 70855, 70855 }, + { 71040, 71086 }, + { 71128, 71131 }, + { 71168, 71215 }, + { 71236, 71236 }, + { 71296, 71338 }, + { 71352, 71352 }, + { 71424, 71450 }, + { 71680, 71723 }, + { 71840, 71903 }, + { 71935, 71935 }, + { 72096, 72103 }, + { 72106, 72144 }, + { 72161, 72161 }, + { 72163, 72163 }, + { 72192, 72192 }, + { 72203, 72242 }, + { 72250, 72250 }, + { 72272, 72272 }, + { 72284, 72329 }, + { 72349, 72349 }, + { 72384, 72440 }, + { 72704, 72712 }, + { 72714, 72750 }, + { 72768, 72768 }, + { 72818, 72847 }, + { 72960, 72966 }, + { 72968, 72969 }, + { 72971, 73008 }, + { 73030, 73030 }, + { 73056, 73061 }, + { 73063, 73064 }, + { 73066, 73097 }, + { 73112, 73112 }, + { 73440, 73458 }, + { 73728, 74649 }, + { 74880, 75075 }, + { 77824, 78894 }, + { 82944, 83526 }, + { 92160, 92728 }, + { 92736, 92766 }, + { 92880, 92909 }, + { 92928, 92975 }, + { 92992, 92995 }, + { 93027, 93047 }, + { 93053, 93071 }, + { 93760, 93823 }, + { 93952, 94026 }, + { 94032, 94032 }, + { 94099, 94111 }, + { 94176, 94177 }, + { 94179, 94179 }, + { 94208, 100343 }, + { 100352, 101106 }, + { 110592, 110878 }, + { 110928, 110930 }, + { 110948, 110951 }, + { 110960, 111355 }, + { 113664, 113770 }, + { 113776, 113788 }, + { 113792, 113800 }, + { 113808, 113817 }, + { 119808, 119892 }, + { 119894, 119964 }, + { 119966, 119967 }, + { 119970, 119970 }, + { 119973, 119974 }, + { 119977, 119980 }, + { 119982, 119993 }, + { 119995, 119995 }, + { 119997, 120003 }, + { 120005, 120069 }, + { 120071, 120074 }, + { 120077, 120084 }, + { 120086, 120092 }, + { 120094, 120121 }, + { 120123, 120126 }, + { 120128, 120132 }, + { 120134, 120134 }, + { 120138, 120144 }, + { 120146, 120485 }, + { 120488, 120512 }, + { 120514, 120538 }, + { 120540, 120570 }, + { 120572, 120596 }, + { 120598, 120628 }, + { 120630, 120654 }, + { 120656, 120686 }, + { 120688, 120712 }, + { 120714, 120744 }, + { 120746, 120770 }, + { 120772, 120779 }, + { 123136, 123180 }, + { 123191, 123197 }, + { 123214, 123214 }, + { 123584, 123627 }, + { 124928, 125124 }, + { 125184, 125251 }, + { 125259, 125259 }, + { 126464, 126467 }, + { 126469, 126495 }, + { 126497, 126498 }, + { 126500, 126500 }, + { 126503, 126503 }, + { 126505, 126514 }, + { 126516, 126519 }, + { 126521, 126521 }, + { 126523, 126523 }, + { 126530, 126530 }, + { 126535, 126535 }, + { 126537, 126537 }, + { 126539, 126539 }, + { 126541, 126543 }, + { 126545, 126546 }, + { 126548, 126548 }, + { 126551, 126551 }, + { 126553, 126553 }, + { 126555, 126555 }, + { 126557, 126557 }, + { 126559, 126559 }, + { 126561, 126562 }, + { 126564, 126564 }, + { 126567, 126570 }, + { 126572, 126578 }, + { 126580, 126583 }, + { 126585, 126588 }, + { 126590, 126590 }, + { 126592, 126601 }, + { 126603, 126619 }, + { 126625, 126627 }, + { 126629, 126633 }, + { 126635, 126651 }, + { 131072, 173782 }, + { 173824, 177972 }, + { 177984, 178205 }, + { 178208, 183969 }, + { 183984, 191456 }, + { 194560, 195101 }, +}; +static const URange16 Ll_range16[] = { + { 97, 122 }, + { 181, 181 }, + { 223, 246 }, + { 248, 255 }, + { 257, 257 }, + { 259, 259 }, + { 261, 261 }, + { 263, 263 }, + { 265, 265 }, + { 267, 267 }, + { 269, 269 }, + { 271, 271 }, + { 273, 273 }, + { 275, 275 }, + { 277, 277 }, + { 279, 279 }, + { 281, 281 }, + { 283, 283 }, + { 285, 285 }, + { 287, 287 }, + { 289, 289 }, + { 291, 291 }, + { 293, 293 }, + { 295, 295 }, + { 297, 297 }, + { 299, 299 }, + { 301, 301 }, + { 303, 303 }, + { 305, 305 }, + { 307, 307 }, + { 309, 309 }, + { 311, 312 }, + { 314, 314 }, + { 316, 316 }, + { 318, 318 }, + { 320, 320 }, + { 322, 322 }, + { 324, 324 }, + { 326, 326 }, + { 328, 329 }, + { 331, 331 }, + { 333, 333 }, + { 335, 335 }, + { 337, 337 }, + { 339, 339 }, + { 341, 341 }, + { 343, 343 }, + { 345, 345 }, + { 347, 347 }, + { 349, 349 }, + { 351, 351 }, + { 353, 353 }, + { 355, 355 }, + { 357, 357 }, + { 359, 359 }, + { 361, 361 }, + { 363, 363 }, + { 365, 365 }, + { 367, 367 }, + { 369, 369 }, + { 371, 371 }, + { 373, 373 }, + { 375, 375 }, + { 378, 378 }, + { 380, 380 }, + { 382, 384 }, + { 387, 387 }, + { 389, 389 }, + { 392, 392 }, + { 396, 397 }, + { 402, 402 }, + { 405, 405 }, + { 409, 411 }, + { 414, 414 }, + { 417, 417 }, + { 419, 419 }, + { 421, 421 }, + { 424, 424 }, + { 426, 427 }, + { 429, 429 }, + { 432, 432 }, + { 436, 436 }, + { 438, 438 }, + { 441, 442 }, + { 445, 447 }, + { 454, 454 }, + { 457, 457 }, + { 460, 460 }, + { 462, 462 }, + { 464, 464 }, + { 466, 466 }, + { 468, 468 }, + { 470, 470 }, + { 472, 472 }, + { 474, 474 }, + { 476, 477 }, + { 479, 479 }, + { 481, 481 }, + { 483, 483 }, + { 485, 485 }, + { 487, 487 }, + { 489, 489 }, + { 491, 491 }, + { 493, 493 }, + { 495, 496 }, + { 499, 499 }, + { 501, 501 }, + { 505, 505 }, + { 507, 507 }, + { 509, 509 }, + { 511, 511 }, + { 513, 513 }, + { 515, 515 }, + { 517, 517 }, + { 519, 519 }, + { 521, 521 }, + { 523, 523 }, + { 525, 525 }, + { 527, 527 }, + { 529, 529 }, + { 531, 531 }, + { 533, 533 }, + { 535, 535 }, + { 537, 537 }, + { 539, 539 }, + { 541, 541 }, + { 543, 543 }, + { 545, 545 }, + { 547, 547 }, + { 549, 549 }, + { 551, 551 }, + { 553, 553 }, + { 555, 555 }, + { 557, 557 }, + { 559, 559 }, + { 561, 561 }, + { 563, 569 }, + { 572, 572 }, + { 575, 576 }, + { 578, 578 }, + { 583, 583 }, + { 585, 585 }, + { 587, 587 }, + { 589, 589 }, + { 591, 659 }, + { 661, 687 }, + { 881, 881 }, + { 883, 883 }, + { 887, 887 }, + { 891, 893 }, + { 912, 912 }, + { 940, 974 }, + { 976, 977 }, + { 981, 983 }, + { 985, 985 }, + { 987, 987 }, + { 989, 989 }, + { 991, 991 }, + { 993, 993 }, + { 995, 995 }, + { 997, 997 }, + { 999, 999 }, + { 1001, 1001 }, + { 1003, 1003 }, + { 1005, 1005 }, + { 1007, 1011 }, + { 1013, 1013 }, + { 1016, 1016 }, + { 1019, 1020 }, + { 1072, 1119 }, + { 1121, 1121 }, + { 1123, 1123 }, + { 1125, 1125 }, + { 1127, 1127 }, + { 1129, 1129 }, + { 1131, 1131 }, + { 1133, 1133 }, + { 1135, 1135 }, + { 1137, 1137 }, + { 1139, 1139 }, + { 1141, 1141 }, + { 1143, 1143 }, + { 1145, 1145 }, + { 1147, 1147 }, + { 1149, 1149 }, + { 1151, 1151 }, + { 1153, 1153 }, + { 1163, 1163 }, + { 1165, 1165 }, + { 1167, 1167 }, + { 1169, 1169 }, + { 1171, 1171 }, + { 1173, 1173 }, + { 1175, 1175 }, + { 1177, 1177 }, + { 1179, 1179 }, + { 1181, 1181 }, + { 1183, 1183 }, + { 1185, 1185 }, + { 1187, 1187 }, + { 1189, 1189 }, + { 1191, 1191 }, + { 1193, 1193 }, + { 1195, 1195 }, + { 1197, 1197 }, + { 1199, 1199 }, + { 1201, 1201 }, + { 1203, 1203 }, + { 1205, 1205 }, + { 1207, 1207 }, + { 1209, 1209 }, + { 1211, 1211 }, + { 1213, 1213 }, + { 1215, 1215 }, + { 1218, 1218 }, + { 1220, 1220 }, + { 1222, 1222 }, + { 1224, 1224 }, + { 1226, 1226 }, + { 1228, 1228 }, + { 1230, 1231 }, + { 1233, 1233 }, + { 1235, 1235 }, + { 1237, 1237 }, + { 1239, 1239 }, + { 1241, 1241 }, + { 1243, 1243 }, + { 1245, 1245 }, + { 1247, 1247 }, + { 1249, 1249 }, + { 1251, 1251 }, + { 1253, 1253 }, + { 1255, 1255 }, + { 1257, 1257 }, + { 1259, 1259 }, + { 1261, 1261 }, + { 1263, 1263 }, + { 1265, 1265 }, + { 1267, 1267 }, + { 1269, 1269 }, + { 1271, 1271 }, + { 1273, 1273 }, + { 1275, 1275 }, + { 1277, 1277 }, + { 1279, 1279 }, + { 1281, 1281 }, + { 1283, 1283 }, + { 1285, 1285 }, + { 1287, 1287 }, + { 1289, 1289 }, + { 1291, 1291 }, + { 1293, 1293 }, + { 1295, 1295 }, + { 1297, 1297 }, + { 1299, 1299 }, + { 1301, 1301 }, + { 1303, 1303 }, + { 1305, 1305 }, + { 1307, 1307 }, + { 1309, 1309 }, + { 1311, 1311 }, + { 1313, 1313 }, + { 1315, 1315 }, + { 1317, 1317 }, + { 1319, 1319 }, + { 1321, 1321 }, + { 1323, 1323 }, + { 1325, 1325 }, + { 1327, 1327 }, + { 1376, 1416 }, + { 4304, 4346 }, + { 4349, 4351 }, + { 5112, 5117 }, + { 7296, 7304 }, + { 7424, 7467 }, + { 7531, 7543 }, + { 7545, 7578 }, + { 7681, 7681 }, + { 7683, 7683 }, + { 7685, 7685 }, + { 7687, 7687 }, + { 7689, 7689 }, + { 7691, 7691 }, + { 7693, 7693 }, + { 7695, 7695 }, + { 7697, 7697 }, + { 7699, 7699 }, + { 7701, 7701 }, + { 7703, 7703 }, + { 7705, 7705 }, + { 7707, 7707 }, + { 7709, 7709 }, + { 7711, 7711 }, + { 7713, 7713 }, + { 7715, 7715 }, + { 7717, 7717 }, + { 7719, 7719 }, + { 7721, 7721 }, + { 7723, 7723 }, + { 7725, 7725 }, + { 7727, 7727 }, + { 7729, 7729 }, + { 7731, 7731 }, + { 7733, 7733 }, + { 7735, 7735 }, + { 7737, 7737 }, + { 7739, 7739 }, + { 7741, 7741 }, + { 7743, 7743 }, + { 7745, 7745 }, + { 7747, 7747 }, + { 7749, 7749 }, + { 7751, 7751 }, + { 7753, 7753 }, + { 7755, 7755 }, + { 7757, 7757 }, + { 7759, 7759 }, + { 7761, 7761 }, + { 7763, 7763 }, + { 7765, 7765 }, + { 7767, 7767 }, + { 7769, 7769 }, + { 7771, 7771 }, + { 7773, 7773 }, + { 7775, 7775 }, + { 7777, 7777 }, + { 7779, 7779 }, + { 7781, 7781 }, + { 7783, 7783 }, + { 7785, 7785 }, + { 7787, 7787 }, + { 7789, 7789 }, + { 7791, 7791 }, + { 7793, 7793 }, + { 7795, 7795 }, + { 7797, 7797 }, + { 7799, 7799 }, + { 7801, 7801 }, + { 7803, 7803 }, + { 7805, 7805 }, + { 7807, 7807 }, + { 7809, 7809 }, + { 7811, 7811 }, + { 7813, 7813 }, + { 7815, 7815 }, + { 7817, 7817 }, + { 7819, 7819 }, + { 7821, 7821 }, + { 7823, 7823 }, + { 7825, 7825 }, + { 7827, 7827 }, + { 7829, 7837 }, + { 7839, 7839 }, + { 7841, 7841 }, + { 7843, 7843 }, + { 7845, 7845 }, + { 7847, 7847 }, + { 7849, 7849 }, + { 7851, 7851 }, + { 7853, 7853 }, + { 7855, 7855 }, + { 7857, 7857 }, + { 7859, 7859 }, + { 7861, 7861 }, + { 7863, 7863 }, + { 7865, 7865 }, + { 7867, 7867 }, + { 7869, 7869 }, + { 7871, 7871 }, + { 7873, 7873 }, + { 7875, 7875 }, + { 7877, 7877 }, + { 7879, 7879 }, + { 7881, 7881 }, + { 7883, 7883 }, + { 7885, 7885 }, + { 7887, 7887 }, + { 7889, 7889 }, + { 7891, 7891 }, + { 7893, 7893 }, + { 7895, 7895 }, + { 7897, 7897 }, + { 7899, 7899 }, + { 7901, 7901 }, + { 7903, 7903 }, + { 7905, 7905 }, + { 7907, 7907 }, + { 7909, 7909 }, + { 7911, 7911 }, + { 7913, 7913 }, + { 7915, 7915 }, + { 7917, 7917 }, + { 7919, 7919 }, + { 7921, 7921 }, + { 7923, 7923 }, + { 7925, 7925 }, + { 7927, 7927 }, + { 7929, 7929 }, + { 7931, 7931 }, + { 7933, 7933 }, + { 7935, 7943 }, + { 7952, 7957 }, + { 7968, 7975 }, + { 7984, 7991 }, + { 8000, 8005 }, + { 8016, 8023 }, + { 8032, 8039 }, + { 8048, 8061 }, + { 8064, 8071 }, + { 8080, 8087 }, + { 8096, 8103 }, + { 8112, 8116 }, + { 8118, 8119 }, + { 8126, 8126 }, + { 8130, 8132 }, + { 8134, 8135 }, + { 8144, 8147 }, + { 8150, 8151 }, + { 8160, 8167 }, + { 8178, 8180 }, + { 8182, 8183 }, + { 8458, 8458 }, + { 8462, 8463 }, + { 8467, 8467 }, + { 8495, 8495 }, + { 8500, 8500 }, + { 8505, 8505 }, + { 8508, 8509 }, + { 8518, 8521 }, + { 8526, 8526 }, + { 8580, 8580 }, + { 11312, 11358 }, + { 11361, 11361 }, + { 11365, 11366 }, + { 11368, 11368 }, + { 11370, 11370 }, + { 11372, 11372 }, + { 11377, 11377 }, + { 11379, 11380 }, + { 11382, 11387 }, + { 11393, 11393 }, + { 11395, 11395 }, + { 11397, 11397 }, + { 11399, 11399 }, + { 11401, 11401 }, + { 11403, 11403 }, + { 11405, 11405 }, + { 11407, 11407 }, + { 11409, 11409 }, + { 11411, 11411 }, + { 11413, 11413 }, + { 11415, 11415 }, + { 11417, 11417 }, + { 11419, 11419 }, + { 11421, 11421 }, + { 11423, 11423 }, + { 11425, 11425 }, + { 11427, 11427 }, + { 11429, 11429 }, + { 11431, 11431 }, + { 11433, 11433 }, + { 11435, 11435 }, + { 11437, 11437 }, + { 11439, 11439 }, + { 11441, 11441 }, + { 11443, 11443 }, + { 11445, 11445 }, + { 11447, 11447 }, + { 11449, 11449 }, + { 11451, 11451 }, + { 11453, 11453 }, + { 11455, 11455 }, + { 11457, 11457 }, + { 11459, 11459 }, + { 11461, 11461 }, + { 11463, 11463 }, + { 11465, 11465 }, + { 11467, 11467 }, + { 11469, 11469 }, + { 11471, 11471 }, + { 11473, 11473 }, + { 11475, 11475 }, + { 11477, 11477 }, + { 11479, 11479 }, + { 11481, 11481 }, + { 11483, 11483 }, + { 11485, 11485 }, + { 11487, 11487 }, + { 11489, 11489 }, + { 11491, 11492 }, + { 11500, 11500 }, + { 11502, 11502 }, + { 11507, 11507 }, + { 11520, 11557 }, + { 11559, 11559 }, + { 11565, 11565 }, + { 42561, 42561 }, + { 42563, 42563 }, + { 42565, 42565 }, + { 42567, 42567 }, + { 42569, 42569 }, + { 42571, 42571 }, + { 42573, 42573 }, + { 42575, 42575 }, + { 42577, 42577 }, + { 42579, 42579 }, + { 42581, 42581 }, + { 42583, 42583 }, + { 42585, 42585 }, + { 42587, 42587 }, + { 42589, 42589 }, + { 42591, 42591 }, + { 42593, 42593 }, + { 42595, 42595 }, + { 42597, 42597 }, + { 42599, 42599 }, + { 42601, 42601 }, + { 42603, 42603 }, + { 42605, 42605 }, + { 42625, 42625 }, + { 42627, 42627 }, + { 42629, 42629 }, + { 42631, 42631 }, + { 42633, 42633 }, + { 42635, 42635 }, + { 42637, 42637 }, + { 42639, 42639 }, + { 42641, 42641 }, + { 42643, 42643 }, + { 42645, 42645 }, + { 42647, 42647 }, + { 42649, 42649 }, + { 42651, 42651 }, + { 42787, 42787 }, + { 42789, 42789 }, + { 42791, 42791 }, + { 42793, 42793 }, + { 42795, 42795 }, + { 42797, 42797 }, + { 42799, 42801 }, + { 42803, 42803 }, + { 42805, 42805 }, + { 42807, 42807 }, + { 42809, 42809 }, + { 42811, 42811 }, + { 42813, 42813 }, + { 42815, 42815 }, + { 42817, 42817 }, + { 42819, 42819 }, + { 42821, 42821 }, + { 42823, 42823 }, + { 42825, 42825 }, + { 42827, 42827 }, + { 42829, 42829 }, + { 42831, 42831 }, + { 42833, 42833 }, + { 42835, 42835 }, + { 42837, 42837 }, + { 42839, 42839 }, + { 42841, 42841 }, + { 42843, 42843 }, + { 42845, 42845 }, + { 42847, 42847 }, + { 42849, 42849 }, + { 42851, 42851 }, + { 42853, 42853 }, + { 42855, 42855 }, + { 42857, 42857 }, + { 42859, 42859 }, + { 42861, 42861 }, + { 42863, 42863 }, + { 42865, 42872 }, + { 42874, 42874 }, + { 42876, 42876 }, + { 42879, 42879 }, + { 42881, 42881 }, + { 42883, 42883 }, + { 42885, 42885 }, + { 42887, 42887 }, + { 42892, 42892 }, + { 42894, 42894 }, + { 42897, 42897 }, + { 42899, 42901 }, + { 42903, 42903 }, + { 42905, 42905 }, + { 42907, 42907 }, + { 42909, 42909 }, + { 42911, 42911 }, + { 42913, 42913 }, + { 42915, 42915 }, + { 42917, 42917 }, + { 42919, 42919 }, + { 42921, 42921 }, + { 42927, 42927 }, + { 42933, 42933 }, + { 42935, 42935 }, + { 42937, 42937 }, + { 42939, 42939 }, + { 42941, 42941 }, + { 42943, 42943 }, + { 42947, 42947 }, + { 43002, 43002 }, + { 43824, 43866 }, + { 43872, 43879 }, + { 43888, 43967 }, + { 64256, 64262 }, + { 64275, 64279 }, + { 65345, 65370 }, +}; +static const URange32 Ll_range32[] = { + { 66600, 66639 }, + { 66776, 66811 }, + { 68800, 68850 }, + { 71872, 71903 }, + { 93792, 93823 }, + { 119834, 119859 }, + { 119886, 119892 }, + { 119894, 119911 }, + { 119938, 119963 }, + { 119990, 119993 }, + { 119995, 119995 }, + { 119997, 120003 }, + { 120005, 120015 }, + { 120042, 120067 }, + { 120094, 120119 }, + { 120146, 120171 }, + { 120198, 120223 }, + { 120250, 120275 }, + { 120302, 120327 }, + { 120354, 120379 }, + { 120406, 120431 }, + { 120458, 120485 }, + { 120514, 120538 }, + { 120540, 120545 }, + { 120572, 120596 }, + { 120598, 120603 }, + { 120630, 120654 }, + { 120656, 120661 }, + { 120688, 120712 }, + { 120714, 120719 }, + { 120746, 120770 }, + { 120772, 120777 }, + { 120779, 120779 }, + { 125218, 125251 }, +}; +static const URange16 Lm_range16[] = { + { 688, 705 }, + { 710, 721 }, + { 736, 740 }, + { 748, 748 }, + { 750, 750 }, + { 884, 884 }, + { 890, 890 }, + { 1369, 1369 }, + { 1600, 1600 }, + { 1765, 1766 }, + { 2036, 2037 }, + { 2042, 2042 }, + { 2074, 2074 }, + { 2084, 2084 }, + { 2088, 2088 }, + { 2417, 2417 }, + { 3654, 3654 }, + { 3782, 3782 }, + { 4348, 4348 }, + { 6103, 6103 }, + { 6211, 6211 }, + { 6823, 6823 }, + { 7288, 7293 }, + { 7468, 7530 }, + { 7544, 7544 }, + { 7579, 7615 }, + { 8305, 8305 }, + { 8319, 8319 }, + { 8336, 8348 }, + { 11388, 11389 }, + { 11631, 11631 }, + { 11823, 11823 }, + { 12293, 12293 }, + { 12337, 12341 }, + { 12347, 12347 }, + { 12445, 12446 }, + { 12540, 12542 }, + { 40981, 40981 }, + { 42232, 42237 }, + { 42508, 42508 }, + { 42623, 42623 }, + { 42652, 42653 }, + { 42775, 42783 }, + { 42864, 42864 }, + { 42888, 42888 }, + { 43000, 43001 }, + { 43471, 43471 }, + { 43494, 43494 }, + { 43632, 43632 }, + { 43741, 43741 }, + { 43763, 43764 }, + { 43868, 43871 }, + { 65392, 65392 }, + { 65438, 65439 }, +}; +static const URange32 Lm_range32[] = { + { 92992, 92995 }, + { 94099, 94111 }, + { 94176, 94177 }, + { 94179, 94179 }, + { 123191, 123197 }, + { 125259, 125259 }, +}; +static const URange16 Lo_range16[] = { + { 170, 170 }, + { 186, 186 }, + { 443, 443 }, + { 448, 451 }, + { 660, 660 }, + { 1488, 1514 }, + { 1519, 1522 }, + { 1568, 1599 }, + { 1601, 1610 }, + { 1646, 1647 }, + { 1649, 1747 }, + { 1749, 1749 }, + { 1774, 1775 }, + { 1786, 1788 }, + { 1791, 1791 }, + { 1808, 1808 }, + { 1810, 1839 }, + { 1869, 1957 }, + { 1969, 1969 }, + { 1994, 2026 }, + { 2048, 2069 }, + { 2112, 2136 }, + { 2144, 2154 }, + { 2208, 2228 }, + { 2230, 2237 }, + { 2308, 2361 }, + { 2365, 2365 }, + { 2384, 2384 }, + { 2392, 2401 }, + { 2418, 2432 }, + { 2437, 2444 }, + { 2447, 2448 }, + { 2451, 2472 }, + { 2474, 2480 }, + { 2482, 2482 }, + { 2486, 2489 }, + { 2493, 2493 }, + { 2510, 2510 }, + { 2524, 2525 }, + { 2527, 2529 }, + { 2544, 2545 }, + { 2556, 2556 }, + { 2565, 2570 }, + { 2575, 2576 }, + { 2579, 2600 }, + { 2602, 2608 }, + { 2610, 2611 }, + { 2613, 2614 }, + { 2616, 2617 }, + { 2649, 2652 }, + { 2654, 2654 }, + { 2674, 2676 }, + { 2693, 2701 }, + { 2703, 2705 }, + { 2707, 2728 }, + { 2730, 2736 }, + { 2738, 2739 }, + { 2741, 2745 }, + { 2749, 2749 }, + { 2768, 2768 }, + { 2784, 2785 }, + { 2809, 2809 }, + { 2821, 2828 }, + { 2831, 2832 }, + { 2835, 2856 }, + { 2858, 2864 }, + { 2866, 2867 }, + { 2869, 2873 }, + { 2877, 2877 }, + { 2908, 2909 }, + { 2911, 2913 }, + { 2929, 2929 }, + { 2947, 2947 }, + { 2949, 2954 }, + { 2958, 2960 }, + { 2962, 2965 }, + { 2969, 2970 }, + { 2972, 2972 }, + { 2974, 2975 }, + { 2979, 2980 }, + { 2984, 2986 }, + { 2990, 3001 }, + { 3024, 3024 }, + { 3077, 3084 }, + { 3086, 3088 }, + { 3090, 3112 }, + { 3114, 3129 }, + { 3133, 3133 }, + { 3160, 3162 }, + { 3168, 3169 }, + { 3200, 3200 }, + { 3205, 3212 }, + { 3214, 3216 }, + { 3218, 3240 }, + { 3242, 3251 }, + { 3253, 3257 }, + { 3261, 3261 }, + { 3294, 3294 }, + { 3296, 3297 }, + { 3313, 3314 }, + { 3333, 3340 }, + { 3342, 3344 }, + { 3346, 3386 }, + { 3389, 3389 }, + { 3406, 3406 }, + { 3412, 3414 }, + { 3423, 3425 }, + { 3450, 3455 }, + { 3461, 3478 }, + { 3482, 3505 }, + { 3507, 3515 }, + { 3517, 3517 }, + { 3520, 3526 }, + { 3585, 3632 }, + { 3634, 3635 }, + { 3648, 3653 }, + { 3713, 3714 }, + { 3716, 3716 }, + { 3718, 3722 }, + { 3724, 3747 }, + { 3749, 3749 }, + { 3751, 3760 }, + { 3762, 3763 }, + { 3773, 3773 }, + { 3776, 3780 }, + { 3804, 3807 }, + { 3840, 3840 }, + { 3904, 3911 }, + { 3913, 3948 }, + { 3976, 3980 }, + { 4096, 4138 }, + { 4159, 4159 }, + { 4176, 4181 }, + { 4186, 4189 }, + { 4193, 4193 }, + { 4197, 4198 }, + { 4206, 4208 }, + { 4213, 4225 }, + { 4238, 4238 }, + { 4352, 4680 }, + { 4682, 4685 }, + { 4688, 4694 }, + { 4696, 4696 }, + { 4698, 4701 }, + { 4704, 4744 }, + { 4746, 4749 }, + { 4752, 4784 }, + { 4786, 4789 }, + { 4792, 4798 }, + { 4800, 4800 }, + { 4802, 4805 }, + { 4808, 4822 }, + { 4824, 4880 }, + { 4882, 4885 }, + { 4888, 4954 }, + { 4992, 5007 }, + { 5121, 5740 }, + { 5743, 5759 }, + { 5761, 5786 }, + { 5792, 5866 }, + { 5873, 5880 }, + { 5888, 5900 }, + { 5902, 5905 }, + { 5920, 5937 }, + { 5952, 5969 }, + { 5984, 5996 }, + { 5998, 6000 }, + { 6016, 6067 }, + { 6108, 6108 }, + { 6176, 6210 }, + { 6212, 6264 }, + { 6272, 6276 }, + { 6279, 6312 }, + { 6314, 6314 }, + { 6320, 6389 }, + { 6400, 6430 }, + { 6480, 6509 }, + { 6512, 6516 }, + { 6528, 6571 }, + { 6576, 6601 }, + { 6656, 6678 }, + { 6688, 6740 }, + { 6917, 6963 }, + { 6981, 6987 }, + { 7043, 7072 }, + { 7086, 7087 }, + { 7098, 7141 }, + { 7168, 7203 }, + { 7245, 7247 }, + { 7258, 7287 }, + { 7401, 7404 }, + { 7406, 7411 }, + { 7413, 7414 }, + { 7418, 7418 }, + { 8501, 8504 }, + { 11568, 11623 }, + { 11648, 11670 }, + { 11680, 11686 }, + { 11688, 11694 }, + { 11696, 11702 }, + { 11704, 11710 }, + { 11712, 11718 }, + { 11720, 11726 }, + { 11728, 11734 }, + { 11736, 11742 }, + { 12294, 12294 }, + { 12348, 12348 }, + { 12353, 12438 }, + { 12447, 12447 }, + { 12449, 12538 }, + { 12543, 12543 }, + { 12549, 12591 }, + { 12593, 12686 }, + { 12704, 12730 }, + { 12784, 12799 }, + { 13312, 19893 }, + { 19968, 40943 }, + { 40960, 40980 }, + { 40982, 42124 }, + { 42192, 42231 }, + { 42240, 42507 }, + { 42512, 42527 }, + { 42538, 42539 }, + { 42606, 42606 }, + { 42656, 42725 }, + { 42895, 42895 }, + { 42999, 42999 }, + { 43003, 43009 }, + { 43011, 43013 }, + { 43015, 43018 }, + { 43020, 43042 }, + { 43072, 43123 }, + { 43138, 43187 }, + { 43250, 43255 }, + { 43259, 43259 }, + { 43261, 43262 }, + { 43274, 43301 }, + { 43312, 43334 }, + { 43360, 43388 }, + { 43396, 43442 }, + { 43488, 43492 }, + { 43495, 43503 }, + { 43514, 43518 }, + { 43520, 43560 }, + { 43584, 43586 }, + { 43588, 43595 }, + { 43616, 43631 }, + { 43633, 43638 }, + { 43642, 43642 }, + { 43646, 43695 }, + { 43697, 43697 }, + { 43701, 43702 }, + { 43705, 43709 }, + { 43712, 43712 }, + { 43714, 43714 }, + { 43739, 43740 }, + { 43744, 43754 }, + { 43762, 43762 }, + { 43777, 43782 }, + { 43785, 43790 }, + { 43793, 43798 }, + { 43808, 43814 }, + { 43816, 43822 }, + { 43968, 44002 }, + { 44032, 55203 }, + { 55216, 55238 }, + { 55243, 55291 }, + { 63744, 64109 }, + { 64112, 64217 }, + { 64285, 64285 }, + { 64287, 64296 }, + { 64298, 64310 }, + { 64312, 64316 }, + { 64318, 64318 }, + { 64320, 64321 }, + { 64323, 64324 }, + { 64326, 64433 }, + { 64467, 64829 }, + { 64848, 64911 }, + { 64914, 64967 }, + { 65008, 65019 }, + { 65136, 65140 }, + { 65142, 65276 }, + { 65382, 65391 }, + { 65393, 65437 }, + { 65440, 65470 }, + { 65474, 65479 }, + { 65482, 65487 }, + { 65490, 65495 }, + { 65498, 65500 }, +}; +static const URange32 Lo_range32[] = { + { 65536, 65547 }, + { 65549, 65574 }, + { 65576, 65594 }, + { 65596, 65597 }, + { 65599, 65613 }, + { 65616, 65629 }, + { 65664, 65786 }, + { 66176, 66204 }, + { 66208, 66256 }, + { 66304, 66335 }, + { 66349, 66368 }, + { 66370, 66377 }, + { 66384, 66421 }, + { 66432, 66461 }, + { 66464, 66499 }, + { 66504, 66511 }, + { 66640, 66717 }, + { 66816, 66855 }, + { 66864, 66915 }, + { 67072, 67382 }, + { 67392, 67413 }, + { 67424, 67431 }, + { 67584, 67589 }, + { 67592, 67592 }, + { 67594, 67637 }, + { 67639, 67640 }, + { 67644, 67644 }, + { 67647, 67669 }, + { 67680, 67702 }, + { 67712, 67742 }, + { 67808, 67826 }, + { 67828, 67829 }, + { 67840, 67861 }, + { 67872, 67897 }, + { 67968, 68023 }, + { 68030, 68031 }, + { 68096, 68096 }, + { 68112, 68115 }, + { 68117, 68119 }, + { 68121, 68149 }, + { 68192, 68220 }, + { 68224, 68252 }, + { 68288, 68295 }, + { 68297, 68324 }, + { 68352, 68405 }, + { 68416, 68437 }, + { 68448, 68466 }, + { 68480, 68497 }, + { 68608, 68680 }, + { 68864, 68899 }, + { 69376, 69404 }, + { 69415, 69415 }, + { 69424, 69445 }, + { 69600, 69622 }, + { 69635, 69687 }, + { 69763, 69807 }, + { 69840, 69864 }, + { 69891, 69926 }, + { 69956, 69956 }, + { 69968, 70002 }, + { 70006, 70006 }, + { 70019, 70066 }, + { 70081, 70084 }, + { 70106, 70106 }, + { 70108, 70108 }, + { 70144, 70161 }, + { 70163, 70187 }, + { 70272, 70278 }, + { 70280, 70280 }, + { 70282, 70285 }, + { 70287, 70301 }, + { 70303, 70312 }, + { 70320, 70366 }, + { 70405, 70412 }, + { 70415, 70416 }, + { 70419, 70440 }, + { 70442, 70448 }, + { 70450, 70451 }, + { 70453, 70457 }, + { 70461, 70461 }, + { 70480, 70480 }, + { 70493, 70497 }, + { 70656, 70708 }, + { 70727, 70730 }, + { 70751, 70751 }, + { 70784, 70831 }, + { 70852, 70853 }, + { 70855, 70855 }, + { 71040, 71086 }, + { 71128, 71131 }, + { 71168, 71215 }, + { 71236, 71236 }, + { 71296, 71338 }, + { 71352, 71352 }, + { 71424, 71450 }, + { 71680, 71723 }, + { 71935, 71935 }, + { 72096, 72103 }, + { 72106, 72144 }, + { 72161, 72161 }, + { 72163, 72163 }, + { 72192, 72192 }, + { 72203, 72242 }, + { 72250, 72250 }, + { 72272, 72272 }, + { 72284, 72329 }, + { 72349, 72349 }, + { 72384, 72440 }, + { 72704, 72712 }, + { 72714, 72750 }, + { 72768, 72768 }, + { 72818, 72847 }, + { 72960, 72966 }, + { 72968, 72969 }, + { 72971, 73008 }, + { 73030, 73030 }, + { 73056, 73061 }, + { 73063, 73064 }, + { 73066, 73097 }, + { 73112, 73112 }, + { 73440, 73458 }, + { 73728, 74649 }, + { 74880, 75075 }, + { 77824, 78894 }, + { 82944, 83526 }, + { 92160, 92728 }, + { 92736, 92766 }, + { 92880, 92909 }, + { 92928, 92975 }, + { 93027, 93047 }, + { 93053, 93071 }, + { 93952, 94026 }, + { 94032, 94032 }, + { 94208, 100343 }, + { 100352, 101106 }, + { 110592, 110878 }, + { 110928, 110930 }, + { 110948, 110951 }, + { 110960, 111355 }, + { 113664, 113770 }, + { 113776, 113788 }, + { 113792, 113800 }, + { 113808, 113817 }, + { 123136, 123180 }, + { 123214, 123214 }, + { 123584, 123627 }, + { 124928, 125124 }, + { 126464, 126467 }, + { 126469, 126495 }, + { 126497, 126498 }, + { 126500, 126500 }, + { 126503, 126503 }, + { 126505, 126514 }, + { 126516, 126519 }, + { 126521, 126521 }, + { 126523, 126523 }, + { 126530, 126530 }, + { 126535, 126535 }, + { 126537, 126537 }, + { 126539, 126539 }, + { 126541, 126543 }, + { 126545, 126546 }, + { 126548, 126548 }, + { 126551, 126551 }, + { 126553, 126553 }, + { 126555, 126555 }, + { 126557, 126557 }, + { 126559, 126559 }, + { 126561, 126562 }, + { 126564, 126564 }, + { 126567, 126570 }, + { 126572, 126578 }, + { 126580, 126583 }, + { 126585, 126588 }, + { 126590, 126590 }, + { 126592, 126601 }, + { 126603, 126619 }, + { 126625, 126627 }, + { 126629, 126633 }, + { 126635, 126651 }, + { 131072, 173782 }, + { 173824, 177972 }, + { 177984, 178205 }, + { 178208, 183969 }, + { 183984, 191456 }, + { 194560, 195101 }, +}; +static const URange16 Lt_range16[] = { + { 453, 453 }, + { 456, 456 }, + { 459, 459 }, + { 498, 498 }, + { 8072, 8079 }, + { 8088, 8095 }, + { 8104, 8111 }, + { 8124, 8124 }, + { 8140, 8140 }, + { 8188, 8188 }, +}; +static const URange16 Lu_range16[] = { + { 65, 90 }, + { 192, 214 }, + { 216, 222 }, + { 256, 256 }, + { 258, 258 }, + { 260, 260 }, + { 262, 262 }, + { 264, 264 }, + { 266, 266 }, + { 268, 268 }, + { 270, 270 }, + { 272, 272 }, + { 274, 274 }, + { 276, 276 }, + { 278, 278 }, + { 280, 280 }, + { 282, 282 }, + { 284, 284 }, + { 286, 286 }, + { 288, 288 }, + { 290, 290 }, + { 292, 292 }, + { 294, 294 }, + { 296, 296 }, + { 298, 298 }, + { 300, 300 }, + { 302, 302 }, + { 304, 304 }, + { 306, 306 }, + { 308, 308 }, + { 310, 310 }, + { 313, 313 }, + { 315, 315 }, + { 317, 317 }, + { 319, 319 }, + { 321, 321 }, + { 323, 323 }, + { 325, 325 }, + { 327, 327 }, + { 330, 330 }, + { 332, 332 }, + { 334, 334 }, + { 336, 336 }, + { 338, 338 }, + { 340, 340 }, + { 342, 342 }, + { 344, 344 }, + { 346, 346 }, + { 348, 348 }, + { 350, 350 }, + { 352, 352 }, + { 354, 354 }, + { 356, 356 }, + { 358, 358 }, + { 360, 360 }, + { 362, 362 }, + { 364, 364 }, + { 366, 366 }, + { 368, 368 }, + { 370, 370 }, + { 372, 372 }, + { 374, 374 }, + { 376, 377 }, + { 379, 379 }, + { 381, 381 }, + { 385, 386 }, + { 388, 388 }, + { 390, 391 }, + { 393, 395 }, + { 398, 401 }, + { 403, 404 }, + { 406, 408 }, + { 412, 413 }, + { 415, 416 }, + { 418, 418 }, + { 420, 420 }, + { 422, 423 }, + { 425, 425 }, + { 428, 428 }, + { 430, 431 }, + { 433, 435 }, + { 437, 437 }, + { 439, 440 }, + { 444, 444 }, + { 452, 452 }, + { 455, 455 }, + { 458, 458 }, + { 461, 461 }, + { 463, 463 }, + { 465, 465 }, + { 467, 467 }, + { 469, 469 }, + { 471, 471 }, + { 473, 473 }, + { 475, 475 }, + { 478, 478 }, + { 480, 480 }, + { 482, 482 }, + { 484, 484 }, + { 486, 486 }, + { 488, 488 }, + { 490, 490 }, + { 492, 492 }, + { 494, 494 }, + { 497, 497 }, + { 500, 500 }, + { 502, 504 }, + { 506, 506 }, + { 508, 508 }, + { 510, 510 }, + { 512, 512 }, + { 514, 514 }, + { 516, 516 }, + { 518, 518 }, + { 520, 520 }, + { 522, 522 }, + { 524, 524 }, + { 526, 526 }, + { 528, 528 }, + { 530, 530 }, + { 532, 532 }, + { 534, 534 }, + { 536, 536 }, + { 538, 538 }, + { 540, 540 }, + { 542, 542 }, + { 544, 544 }, + { 546, 546 }, + { 548, 548 }, + { 550, 550 }, + { 552, 552 }, + { 554, 554 }, + { 556, 556 }, + { 558, 558 }, + { 560, 560 }, + { 562, 562 }, + { 570, 571 }, + { 573, 574 }, + { 577, 577 }, + { 579, 582 }, + { 584, 584 }, + { 586, 586 }, + { 588, 588 }, + { 590, 590 }, + { 880, 880 }, + { 882, 882 }, + { 886, 886 }, + { 895, 895 }, + { 902, 902 }, + { 904, 906 }, + { 908, 908 }, + { 910, 911 }, + { 913, 929 }, + { 931, 939 }, + { 975, 975 }, + { 978, 980 }, + { 984, 984 }, + { 986, 986 }, + { 988, 988 }, + { 990, 990 }, + { 992, 992 }, + { 994, 994 }, + { 996, 996 }, + { 998, 998 }, + { 1000, 1000 }, + { 1002, 1002 }, + { 1004, 1004 }, + { 1006, 1006 }, + { 1012, 1012 }, + { 1015, 1015 }, + { 1017, 1018 }, + { 1021, 1071 }, + { 1120, 1120 }, + { 1122, 1122 }, + { 1124, 1124 }, + { 1126, 1126 }, + { 1128, 1128 }, + { 1130, 1130 }, + { 1132, 1132 }, + { 1134, 1134 }, + { 1136, 1136 }, + { 1138, 1138 }, + { 1140, 1140 }, + { 1142, 1142 }, + { 1144, 1144 }, + { 1146, 1146 }, + { 1148, 1148 }, + { 1150, 1150 }, + { 1152, 1152 }, + { 1162, 1162 }, + { 1164, 1164 }, + { 1166, 1166 }, + { 1168, 1168 }, + { 1170, 1170 }, + { 1172, 1172 }, + { 1174, 1174 }, + { 1176, 1176 }, + { 1178, 1178 }, + { 1180, 1180 }, + { 1182, 1182 }, + { 1184, 1184 }, + { 1186, 1186 }, + { 1188, 1188 }, + { 1190, 1190 }, + { 1192, 1192 }, + { 1194, 1194 }, + { 1196, 1196 }, + { 1198, 1198 }, + { 1200, 1200 }, + { 1202, 1202 }, + { 1204, 1204 }, + { 1206, 1206 }, + { 1208, 1208 }, + { 1210, 1210 }, + { 1212, 1212 }, + { 1214, 1214 }, + { 1216, 1217 }, + { 1219, 1219 }, + { 1221, 1221 }, + { 1223, 1223 }, + { 1225, 1225 }, + { 1227, 1227 }, + { 1229, 1229 }, + { 1232, 1232 }, + { 1234, 1234 }, + { 1236, 1236 }, + { 1238, 1238 }, + { 1240, 1240 }, + { 1242, 1242 }, + { 1244, 1244 }, + { 1246, 1246 }, + { 1248, 1248 }, + { 1250, 1250 }, + { 1252, 1252 }, + { 1254, 1254 }, + { 1256, 1256 }, + { 1258, 1258 }, + { 1260, 1260 }, + { 1262, 1262 }, + { 1264, 1264 }, + { 1266, 1266 }, + { 1268, 1268 }, + { 1270, 1270 }, + { 1272, 1272 }, + { 1274, 1274 }, + { 1276, 1276 }, + { 1278, 1278 }, + { 1280, 1280 }, + { 1282, 1282 }, + { 1284, 1284 }, + { 1286, 1286 }, + { 1288, 1288 }, + { 1290, 1290 }, + { 1292, 1292 }, + { 1294, 1294 }, + { 1296, 1296 }, + { 1298, 1298 }, + { 1300, 1300 }, + { 1302, 1302 }, + { 1304, 1304 }, + { 1306, 1306 }, + { 1308, 1308 }, + { 1310, 1310 }, + { 1312, 1312 }, + { 1314, 1314 }, + { 1316, 1316 }, + { 1318, 1318 }, + { 1320, 1320 }, + { 1322, 1322 }, + { 1324, 1324 }, + { 1326, 1326 }, + { 1329, 1366 }, + { 4256, 4293 }, + { 4295, 4295 }, + { 4301, 4301 }, + { 5024, 5109 }, + { 7312, 7354 }, + { 7357, 7359 }, + { 7680, 7680 }, + { 7682, 7682 }, + { 7684, 7684 }, + { 7686, 7686 }, + { 7688, 7688 }, + { 7690, 7690 }, + { 7692, 7692 }, + { 7694, 7694 }, + { 7696, 7696 }, + { 7698, 7698 }, + { 7700, 7700 }, + { 7702, 7702 }, + { 7704, 7704 }, + { 7706, 7706 }, + { 7708, 7708 }, + { 7710, 7710 }, + { 7712, 7712 }, + { 7714, 7714 }, + { 7716, 7716 }, + { 7718, 7718 }, + { 7720, 7720 }, + { 7722, 7722 }, + { 7724, 7724 }, + { 7726, 7726 }, + { 7728, 7728 }, + { 7730, 7730 }, + { 7732, 7732 }, + { 7734, 7734 }, + { 7736, 7736 }, + { 7738, 7738 }, + { 7740, 7740 }, + { 7742, 7742 }, + { 7744, 7744 }, + { 7746, 7746 }, + { 7748, 7748 }, + { 7750, 7750 }, + { 7752, 7752 }, + { 7754, 7754 }, + { 7756, 7756 }, + { 7758, 7758 }, + { 7760, 7760 }, + { 7762, 7762 }, + { 7764, 7764 }, + { 7766, 7766 }, + { 7768, 7768 }, + { 7770, 7770 }, + { 7772, 7772 }, + { 7774, 7774 }, + { 7776, 7776 }, + { 7778, 7778 }, + { 7780, 7780 }, + { 7782, 7782 }, + { 7784, 7784 }, + { 7786, 7786 }, + { 7788, 7788 }, + { 7790, 7790 }, + { 7792, 7792 }, + { 7794, 7794 }, + { 7796, 7796 }, + { 7798, 7798 }, + { 7800, 7800 }, + { 7802, 7802 }, + { 7804, 7804 }, + { 7806, 7806 }, + { 7808, 7808 }, + { 7810, 7810 }, + { 7812, 7812 }, + { 7814, 7814 }, + { 7816, 7816 }, + { 7818, 7818 }, + { 7820, 7820 }, + { 7822, 7822 }, + { 7824, 7824 }, + { 7826, 7826 }, + { 7828, 7828 }, + { 7838, 7838 }, + { 7840, 7840 }, + { 7842, 7842 }, + { 7844, 7844 }, + { 7846, 7846 }, + { 7848, 7848 }, + { 7850, 7850 }, + { 7852, 7852 }, + { 7854, 7854 }, + { 7856, 7856 }, + { 7858, 7858 }, + { 7860, 7860 }, + { 7862, 7862 }, + { 7864, 7864 }, + { 7866, 7866 }, + { 7868, 7868 }, + { 7870, 7870 }, + { 7872, 7872 }, + { 7874, 7874 }, + { 7876, 7876 }, + { 7878, 7878 }, + { 7880, 7880 }, + { 7882, 7882 }, + { 7884, 7884 }, + { 7886, 7886 }, + { 7888, 7888 }, + { 7890, 7890 }, + { 7892, 7892 }, + { 7894, 7894 }, + { 7896, 7896 }, + { 7898, 7898 }, + { 7900, 7900 }, + { 7902, 7902 }, + { 7904, 7904 }, + { 7906, 7906 }, + { 7908, 7908 }, + { 7910, 7910 }, + { 7912, 7912 }, + { 7914, 7914 }, + { 7916, 7916 }, + { 7918, 7918 }, + { 7920, 7920 }, + { 7922, 7922 }, + { 7924, 7924 }, + { 7926, 7926 }, + { 7928, 7928 }, + { 7930, 7930 }, + { 7932, 7932 }, + { 7934, 7934 }, + { 7944, 7951 }, + { 7960, 7965 }, + { 7976, 7983 }, + { 7992, 7999 }, + { 8008, 8013 }, + { 8025, 8025 }, + { 8027, 8027 }, + { 8029, 8029 }, + { 8031, 8031 }, + { 8040, 8047 }, + { 8120, 8123 }, + { 8136, 8139 }, + { 8152, 8155 }, + { 8168, 8172 }, + { 8184, 8187 }, + { 8450, 8450 }, + { 8455, 8455 }, + { 8459, 8461 }, + { 8464, 8466 }, + { 8469, 8469 }, + { 8473, 8477 }, + { 8484, 8484 }, + { 8486, 8486 }, + { 8488, 8488 }, + { 8490, 8493 }, + { 8496, 8499 }, + { 8510, 8511 }, + { 8517, 8517 }, + { 8579, 8579 }, + { 11264, 11310 }, + { 11360, 11360 }, + { 11362, 11364 }, + { 11367, 11367 }, + { 11369, 11369 }, + { 11371, 11371 }, + { 11373, 11376 }, + { 11378, 11378 }, + { 11381, 11381 }, + { 11390, 11392 }, + { 11394, 11394 }, + { 11396, 11396 }, + { 11398, 11398 }, + { 11400, 11400 }, + { 11402, 11402 }, + { 11404, 11404 }, + { 11406, 11406 }, + { 11408, 11408 }, + { 11410, 11410 }, + { 11412, 11412 }, + { 11414, 11414 }, + { 11416, 11416 }, + { 11418, 11418 }, + { 11420, 11420 }, + { 11422, 11422 }, + { 11424, 11424 }, + { 11426, 11426 }, + { 11428, 11428 }, + { 11430, 11430 }, + { 11432, 11432 }, + { 11434, 11434 }, + { 11436, 11436 }, + { 11438, 11438 }, + { 11440, 11440 }, + { 11442, 11442 }, + { 11444, 11444 }, + { 11446, 11446 }, + { 11448, 11448 }, + { 11450, 11450 }, + { 11452, 11452 }, + { 11454, 11454 }, + { 11456, 11456 }, + { 11458, 11458 }, + { 11460, 11460 }, + { 11462, 11462 }, + { 11464, 11464 }, + { 11466, 11466 }, + { 11468, 11468 }, + { 11470, 11470 }, + { 11472, 11472 }, + { 11474, 11474 }, + { 11476, 11476 }, + { 11478, 11478 }, + { 11480, 11480 }, + { 11482, 11482 }, + { 11484, 11484 }, + { 11486, 11486 }, + { 11488, 11488 }, + { 11490, 11490 }, + { 11499, 11499 }, + { 11501, 11501 }, + { 11506, 11506 }, + { 42560, 42560 }, + { 42562, 42562 }, + { 42564, 42564 }, + { 42566, 42566 }, + { 42568, 42568 }, + { 42570, 42570 }, + { 42572, 42572 }, + { 42574, 42574 }, + { 42576, 42576 }, + { 42578, 42578 }, + { 42580, 42580 }, + { 42582, 42582 }, + { 42584, 42584 }, + { 42586, 42586 }, + { 42588, 42588 }, + { 42590, 42590 }, + { 42592, 42592 }, + { 42594, 42594 }, + { 42596, 42596 }, + { 42598, 42598 }, + { 42600, 42600 }, + { 42602, 42602 }, + { 42604, 42604 }, + { 42624, 42624 }, + { 42626, 42626 }, + { 42628, 42628 }, + { 42630, 42630 }, + { 42632, 42632 }, + { 42634, 42634 }, + { 42636, 42636 }, + { 42638, 42638 }, + { 42640, 42640 }, + { 42642, 42642 }, + { 42644, 42644 }, + { 42646, 42646 }, + { 42648, 42648 }, + { 42650, 42650 }, + { 42786, 42786 }, + { 42788, 42788 }, + { 42790, 42790 }, + { 42792, 42792 }, + { 42794, 42794 }, + { 42796, 42796 }, + { 42798, 42798 }, + { 42802, 42802 }, + { 42804, 42804 }, + { 42806, 42806 }, + { 42808, 42808 }, + { 42810, 42810 }, + { 42812, 42812 }, + { 42814, 42814 }, + { 42816, 42816 }, + { 42818, 42818 }, + { 42820, 42820 }, + { 42822, 42822 }, + { 42824, 42824 }, + { 42826, 42826 }, + { 42828, 42828 }, + { 42830, 42830 }, + { 42832, 42832 }, + { 42834, 42834 }, + { 42836, 42836 }, + { 42838, 42838 }, + { 42840, 42840 }, + { 42842, 42842 }, + { 42844, 42844 }, + { 42846, 42846 }, + { 42848, 42848 }, + { 42850, 42850 }, + { 42852, 42852 }, + { 42854, 42854 }, + { 42856, 42856 }, + { 42858, 42858 }, + { 42860, 42860 }, + { 42862, 42862 }, + { 42873, 42873 }, + { 42875, 42875 }, + { 42877, 42878 }, + { 42880, 42880 }, + { 42882, 42882 }, + { 42884, 42884 }, + { 42886, 42886 }, + { 42891, 42891 }, + { 42893, 42893 }, + { 42896, 42896 }, + { 42898, 42898 }, + { 42902, 42902 }, + { 42904, 42904 }, + { 42906, 42906 }, + { 42908, 42908 }, + { 42910, 42910 }, + { 42912, 42912 }, + { 42914, 42914 }, + { 42916, 42916 }, + { 42918, 42918 }, + { 42920, 42920 }, + { 42922, 42926 }, + { 42928, 42932 }, + { 42934, 42934 }, + { 42936, 42936 }, + { 42938, 42938 }, + { 42940, 42940 }, + { 42942, 42942 }, + { 42946, 42946 }, + { 42948, 42950 }, + { 65313, 65338 }, +}; +static const URange32 Lu_range32[] = { + { 66560, 66599 }, + { 66736, 66771 }, + { 68736, 68786 }, + { 71840, 71871 }, + { 93760, 93791 }, + { 119808, 119833 }, + { 119860, 119885 }, + { 119912, 119937 }, + { 119964, 119964 }, + { 119966, 119967 }, + { 119970, 119970 }, + { 119973, 119974 }, + { 119977, 119980 }, + { 119982, 119989 }, + { 120016, 120041 }, + { 120068, 120069 }, + { 120071, 120074 }, + { 120077, 120084 }, + { 120086, 120092 }, + { 120120, 120121 }, + { 120123, 120126 }, + { 120128, 120132 }, + { 120134, 120134 }, + { 120138, 120144 }, + { 120172, 120197 }, + { 120224, 120249 }, + { 120276, 120301 }, + { 120328, 120353 }, + { 120380, 120405 }, + { 120432, 120457 }, + { 120488, 120512 }, + { 120546, 120570 }, + { 120604, 120628 }, + { 120662, 120686 }, + { 120720, 120744 }, + { 120778, 120778 }, + { 125184, 125217 }, +}; +static const URange16 M_range16[] = { + { 768, 879 }, + { 1155, 1161 }, + { 1425, 1469 }, + { 1471, 1471 }, + { 1473, 1474 }, + { 1476, 1477 }, + { 1479, 1479 }, + { 1552, 1562 }, + { 1611, 1631 }, + { 1648, 1648 }, + { 1750, 1756 }, + { 1759, 1764 }, + { 1767, 1768 }, + { 1770, 1773 }, + { 1809, 1809 }, + { 1840, 1866 }, + { 1958, 1968 }, + { 2027, 2035 }, + { 2045, 2045 }, + { 2070, 2073 }, + { 2075, 2083 }, + { 2085, 2087 }, + { 2089, 2093 }, + { 2137, 2139 }, + { 2259, 2273 }, + { 2275, 2307 }, + { 2362, 2364 }, + { 2366, 2383 }, + { 2385, 2391 }, + { 2402, 2403 }, + { 2433, 2435 }, + { 2492, 2492 }, + { 2494, 2500 }, + { 2503, 2504 }, + { 2507, 2509 }, + { 2519, 2519 }, + { 2530, 2531 }, + { 2558, 2558 }, + { 2561, 2563 }, + { 2620, 2620 }, + { 2622, 2626 }, + { 2631, 2632 }, + { 2635, 2637 }, + { 2641, 2641 }, + { 2672, 2673 }, + { 2677, 2677 }, + { 2689, 2691 }, + { 2748, 2748 }, + { 2750, 2757 }, + { 2759, 2761 }, + { 2763, 2765 }, + { 2786, 2787 }, + { 2810, 2815 }, + { 2817, 2819 }, + { 2876, 2876 }, + { 2878, 2884 }, + { 2887, 2888 }, + { 2891, 2893 }, + { 2902, 2903 }, + { 2914, 2915 }, + { 2946, 2946 }, + { 3006, 3010 }, + { 3014, 3016 }, + { 3018, 3021 }, + { 3031, 3031 }, + { 3072, 3076 }, + { 3134, 3140 }, + { 3142, 3144 }, + { 3146, 3149 }, + { 3157, 3158 }, + { 3170, 3171 }, + { 3201, 3203 }, + { 3260, 3260 }, + { 3262, 3268 }, + { 3270, 3272 }, + { 3274, 3277 }, + { 3285, 3286 }, + { 3298, 3299 }, + { 3328, 3331 }, + { 3387, 3388 }, + { 3390, 3396 }, + { 3398, 3400 }, + { 3402, 3405 }, + { 3415, 3415 }, + { 3426, 3427 }, + { 3458, 3459 }, + { 3530, 3530 }, + { 3535, 3540 }, + { 3542, 3542 }, + { 3544, 3551 }, + { 3570, 3571 }, + { 3633, 3633 }, + { 3636, 3642 }, + { 3655, 3662 }, + { 3761, 3761 }, + { 3764, 3772 }, + { 3784, 3789 }, + { 3864, 3865 }, + { 3893, 3893 }, + { 3895, 3895 }, + { 3897, 3897 }, + { 3902, 3903 }, + { 3953, 3972 }, + { 3974, 3975 }, + { 3981, 3991 }, + { 3993, 4028 }, + { 4038, 4038 }, + { 4139, 4158 }, + { 4182, 4185 }, + { 4190, 4192 }, + { 4194, 4196 }, + { 4199, 4205 }, + { 4209, 4212 }, + { 4226, 4237 }, + { 4239, 4239 }, + { 4250, 4253 }, + { 4957, 4959 }, + { 5906, 5908 }, + { 5938, 5940 }, + { 5970, 5971 }, + { 6002, 6003 }, + { 6068, 6099 }, + { 6109, 6109 }, + { 6155, 6157 }, + { 6277, 6278 }, + { 6313, 6313 }, + { 6432, 6443 }, + { 6448, 6459 }, + { 6679, 6683 }, + { 6741, 6750 }, + { 6752, 6780 }, + { 6783, 6783 }, + { 6832, 6846 }, + { 6912, 6916 }, + { 6964, 6980 }, + { 7019, 7027 }, + { 7040, 7042 }, + { 7073, 7085 }, + { 7142, 7155 }, + { 7204, 7223 }, + { 7376, 7378 }, + { 7380, 7400 }, + { 7405, 7405 }, + { 7412, 7412 }, + { 7415, 7417 }, + { 7616, 7673 }, + { 7675, 7679 }, + { 8400, 8432 }, + { 11503, 11505 }, + { 11647, 11647 }, + { 11744, 11775 }, + { 12330, 12335 }, + { 12441, 12442 }, + { 42607, 42610 }, + { 42612, 42621 }, + { 42654, 42655 }, + { 42736, 42737 }, + { 43010, 43010 }, + { 43014, 43014 }, + { 43019, 43019 }, + { 43043, 43047 }, + { 43136, 43137 }, + { 43188, 43205 }, + { 43232, 43249 }, + { 43263, 43263 }, + { 43302, 43309 }, + { 43335, 43347 }, + { 43392, 43395 }, + { 43443, 43456 }, + { 43493, 43493 }, + { 43561, 43574 }, + { 43587, 43587 }, + { 43596, 43597 }, + { 43643, 43645 }, + { 43696, 43696 }, + { 43698, 43700 }, + { 43703, 43704 }, + { 43710, 43711 }, + { 43713, 43713 }, + { 43755, 43759 }, + { 43765, 43766 }, + { 44003, 44010 }, + { 44012, 44013 }, + { 64286, 64286 }, + { 65024, 65039 }, + { 65056, 65071 }, +}; +static const URange32 M_range32[] = { + { 66045, 66045 }, + { 66272, 66272 }, + { 66422, 66426 }, + { 68097, 68099 }, + { 68101, 68102 }, + { 68108, 68111 }, + { 68152, 68154 }, + { 68159, 68159 }, + { 68325, 68326 }, + { 68900, 68903 }, + { 69446, 69456 }, + { 69632, 69634 }, + { 69688, 69702 }, + { 69759, 69762 }, + { 69808, 69818 }, + { 69888, 69890 }, + { 69927, 69940 }, + { 69957, 69958 }, + { 70003, 70003 }, + { 70016, 70018 }, + { 70067, 70080 }, + { 70089, 70092 }, + { 70188, 70199 }, + { 70206, 70206 }, + { 70367, 70378 }, + { 70400, 70403 }, + { 70459, 70460 }, + { 70462, 70468 }, + { 70471, 70472 }, + { 70475, 70477 }, + { 70487, 70487 }, + { 70498, 70499 }, + { 70502, 70508 }, + { 70512, 70516 }, + { 70709, 70726 }, + { 70750, 70750 }, + { 70832, 70851 }, + { 71087, 71093 }, + { 71096, 71104 }, + { 71132, 71133 }, + { 71216, 71232 }, + { 71339, 71351 }, + { 71453, 71467 }, + { 71724, 71738 }, + { 72145, 72151 }, + { 72154, 72160 }, + { 72164, 72164 }, + { 72193, 72202 }, + { 72243, 72249 }, + { 72251, 72254 }, + { 72263, 72263 }, + { 72273, 72283 }, + { 72330, 72345 }, + { 72751, 72758 }, + { 72760, 72767 }, + { 72850, 72871 }, + { 72873, 72886 }, + { 73009, 73014 }, + { 73018, 73018 }, + { 73020, 73021 }, + { 73023, 73029 }, + { 73031, 73031 }, + { 73098, 73102 }, + { 73104, 73105 }, + { 73107, 73111 }, + { 73459, 73462 }, + { 92912, 92916 }, + { 92976, 92982 }, + { 94031, 94031 }, + { 94033, 94087 }, + { 94095, 94098 }, + { 113821, 113822 }, + { 119141, 119145 }, + { 119149, 119154 }, + { 119163, 119170 }, + { 119173, 119179 }, + { 119210, 119213 }, + { 119362, 119364 }, + { 121344, 121398 }, + { 121403, 121452 }, + { 121461, 121461 }, + { 121476, 121476 }, + { 121499, 121503 }, + { 121505, 121519 }, + { 122880, 122886 }, + { 122888, 122904 }, + { 122907, 122913 }, + { 122915, 122916 }, + { 122918, 122922 }, + { 123184, 123190 }, + { 123628, 123631 }, + { 125136, 125142 }, + { 125252, 125258 }, + { 917760, 917999 }, +}; +static const URange16 Mc_range16[] = { + { 2307, 2307 }, + { 2363, 2363 }, + { 2366, 2368 }, + { 2377, 2380 }, + { 2382, 2383 }, + { 2434, 2435 }, + { 2494, 2496 }, + { 2503, 2504 }, + { 2507, 2508 }, + { 2519, 2519 }, + { 2563, 2563 }, + { 2622, 2624 }, + { 2691, 2691 }, + { 2750, 2752 }, + { 2761, 2761 }, + { 2763, 2764 }, + { 2818, 2819 }, + { 2878, 2878 }, + { 2880, 2880 }, + { 2887, 2888 }, + { 2891, 2892 }, + { 2903, 2903 }, + { 3006, 3007 }, + { 3009, 3010 }, + { 3014, 3016 }, + { 3018, 3020 }, + { 3031, 3031 }, + { 3073, 3075 }, + { 3137, 3140 }, + { 3202, 3203 }, + { 3262, 3262 }, + { 3264, 3268 }, + { 3271, 3272 }, + { 3274, 3275 }, + { 3285, 3286 }, + { 3330, 3331 }, + { 3390, 3392 }, + { 3398, 3400 }, + { 3402, 3404 }, + { 3415, 3415 }, + { 3458, 3459 }, + { 3535, 3537 }, + { 3544, 3551 }, + { 3570, 3571 }, + { 3902, 3903 }, + { 3967, 3967 }, + { 4139, 4140 }, + { 4145, 4145 }, + { 4152, 4152 }, + { 4155, 4156 }, + { 4182, 4183 }, + { 4194, 4196 }, + { 4199, 4205 }, + { 4227, 4228 }, + { 4231, 4236 }, + { 4239, 4239 }, + { 4250, 4252 }, + { 6070, 6070 }, + { 6078, 6085 }, + { 6087, 6088 }, + { 6435, 6438 }, + { 6441, 6443 }, + { 6448, 6449 }, + { 6451, 6456 }, + { 6681, 6682 }, + { 6741, 6741 }, + { 6743, 6743 }, + { 6753, 6753 }, + { 6755, 6756 }, + { 6765, 6770 }, + { 6916, 6916 }, + { 6965, 6965 }, + { 6971, 6971 }, + { 6973, 6977 }, + { 6979, 6980 }, + { 7042, 7042 }, + { 7073, 7073 }, + { 7078, 7079 }, + { 7082, 7082 }, + { 7143, 7143 }, + { 7146, 7148 }, + { 7150, 7150 }, + { 7154, 7155 }, + { 7204, 7211 }, + { 7220, 7221 }, + { 7393, 7393 }, + { 7415, 7415 }, + { 12334, 12335 }, + { 43043, 43044 }, + { 43047, 43047 }, + { 43136, 43137 }, + { 43188, 43203 }, + { 43346, 43347 }, + { 43395, 43395 }, + { 43444, 43445 }, + { 43450, 43451 }, + { 43454, 43456 }, + { 43567, 43568 }, + { 43571, 43572 }, + { 43597, 43597 }, + { 43643, 43643 }, + { 43645, 43645 }, + { 43755, 43755 }, + { 43758, 43759 }, + { 43765, 43765 }, + { 44003, 44004 }, + { 44006, 44007 }, + { 44009, 44010 }, + { 44012, 44012 }, +}; +static const URange32 Mc_range32[] = { + { 69632, 69632 }, + { 69634, 69634 }, + { 69762, 69762 }, + { 69808, 69810 }, + { 69815, 69816 }, + { 69932, 69932 }, + { 69957, 69958 }, + { 70018, 70018 }, + { 70067, 70069 }, + { 70079, 70080 }, + { 70188, 70190 }, + { 70194, 70195 }, + { 70197, 70197 }, + { 70368, 70370 }, + { 70402, 70403 }, + { 70462, 70463 }, + { 70465, 70468 }, + { 70471, 70472 }, + { 70475, 70477 }, + { 70487, 70487 }, + { 70498, 70499 }, + { 70709, 70711 }, + { 70720, 70721 }, + { 70725, 70725 }, + { 70832, 70834 }, + { 70841, 70841 }, + { 70843, 70846 }, + { 70849, 70849 }, + { 71087, 71089 }, + { 71096, 71099 }, + { 71102, 71102 }, + { 71216, 71218 }, + { 71227, 71228 }, + { 71230, 71230 }, + { 71340, 71340 }, + { 71342, 71343 }, + { 71350, 71350 }, + { 71456, 71457 }, + { 71462, 71462 }, + { 71724, 71726 }, + { 71736, 71736 }, + { 72145, 72147 }, + { 72156, 72159 }, + { 72164, 72164 }, + { 72249, 72249 }, + { 72279, 72280 }, + { 72343, 72343 }, + { 72751, 72751 }, + { 72766, 72766 }, + { 72873, 72873 }, + { 72881, 72881 }, + { 72884, 72884 }, + { 73098, 73102 }, + { 73107, 73108 }, + { 73110, 73110 }, + { 73461, 73462 }, + { 94033, 94087 }, + { 119141, 119142 }, + { 119149, 119154 }, +}; +static const URange16 Me_range16[] = { + { 1160, 1161 }, + { 6846, 6846 }, + { 8413, 8416 }, + { 8418, 8420 }, + { 42608, 42610 }, +}; +static const URange16 Mn_range16[] = { + { 768, 879 }, + { 1155, 1159 }, + { 1425, 1469 }, + { 1471, 1471 }, + { 1473, 1474 }, + { 1476, 1477 }, + { 1479, 1479 }, + { 1552, 1562 }, + { 1611, 1631 }, + { 1648, 1648 }, + { 1750, 1756 }, + { 1759, 1764 }, + { 1767, 1768 }, + { 1770, 1773 }, + { 1809, 1809 }, + { 1840, 1866 }, + { 1958, 1968 }, + { 2027, 2035 }, + { 2045, 2045 }, + { 2070, 2073 }, + { 2075, 2083 }, + { 2085, 2087 }, + { 2089, 2093 }, + { 2137, 2139 }, + { 2259, 2273 }, + { 2275, 2306 }, + { 2362, 2362 }, + { 2364, 2364 }, + { 2369, 2376 }, + { 2381, 2381 }, + { 2385, 2391 }, + { 2402, 2403 }, + { 2433, 2433 }, + { 2492, 2492 }, + { 2497, 2500 }, + { 2509, 2509 }, + { 2530, 2531 }, + { 2558, 2558 }, + { 2561, 2562 }, + { 2620, 2620 }, + { 2625, 2626 }, + { 2631, 2632 }, + { 2635, 2637 }, + { 2641, 2641 }, + { 2672, 2673 }, + { 2677, 2677 }, + { 2689, 2690 }, + { 2748, 2748 }, + { 2753, 2757 }, + { 2759, 2760 }, + { 2765, 2765 }, + { 2786, 2787 }, + { 2810, 2815 }, + { 2817, 2817 }, + { 2876, 2876 }, + { 2879, 2879 }, + { 2881, 2884 }, + { 2893, 2893 }, + { 2902, 2902 }, + { 2914, 2915 }, + { 2946, 2946 }, + { 3008, 3008 }, + { 3021, 3021 }, + { 3072, 3072 }, + { 3076, 3076 }, + { 3134, 3136 }, + { 3142, 3144 }, + { 3146, 3149 }, + { 3157, 3158 }, + { 3170, 3171 }, + { 3201, 3201 }, + { 3260, 3260 }, + { 3263, 3263 }, + { 3270, 3270 }, + { 3276, 3277 }, + { 3298, 3299 }, + { 3328, 3329 }, + { 3387, 3388 }, + { 3393, 3396 }, + { 3405, 3405 }, + { 3426, 3427 }, + { 3530, 3530 }, + { 3538, 3540 }, + { 3542, 3542 }, + { 3633, 3633 }, + { 3636, 3642 }, + { 3655, 3662 }, + { 3761, 3761 }, + { 3764, 3772 }, + { 3784, 3789 }, + { 3864, 3865 }, + { 3893, 3893 }, + { 3895, 3895 }, + { 3897, 3897 }, + { 3953, 3966 }, + { 3968, 3972 }, + { 3974, 3975 }, + { 3981, 3991 }, + { 3993, 4028 }, + { 4038, 4038 }, + { 4141, 4144 }, + { 4146, 4151 }, + { 4153, 4154 }, + { 4157, 4158 }, + { 4184, 4185 }, + { 4190, 4192 }, + { 4209, 4212 }, + { 4226, 4226 }, + { 4229, 4230 }, + { 4237, 4237 }, + { 4253, 4253 }, + { 4957, 4959 }, + { 5906, 5908 }, + { 5938, 5940 }, + { 5970, 5971 }, + { 6002, 6003 }, + { 6068, 6069 }, + { 6071, 6077 }, + { 6086, 6086 }, + { 6089, 6099 }, + { 6109, 6109 }, + { 6155, 6157 }, + { 6277, 6278 }, + { 6313, 6313 }, + { 6432, 6434 }, + { 6439, 6440 }, + { 6450, 6450 }, + { 6457, 6459 }, + { 6679, 6680 }, + { 6683, 6683 }, + { 6742, 6742 }, + { 6744, 6750 }, + { 6752, 6752 }, + { 6754, 6754 }, + { 6757, 6764 }, + { 6771, 6780 }, + { 6783, 6783 }, + { 6832, 6845 }, + { 6912, 6915 }, + { 6964, 6964 }, + { 6966, 6970 }, + { 6972, 6972 }, + { 6978, 6978 }, + { 7019, 7027 }, + { 7040, 7041 }, + { 7074, 7077 }, + { 7080, 7081 }, + { 7083, 7085 }, + { 7142, 7142 }, + { 7144, 7145 }, + { 7149, 7149 }, + { 7151, 7153 }, + { 7212, 7219 }, + { 7222, 7223 }, + { 7376, 7378 }, + { 7380, 7392 }, + { 7394, 7400 }, + { 7405, 7405 }, + { 7412, 7412 }, + { 7416, 7417 }, + { 7616, 7673 }, + { 7675, 7679 }, + { 8400, 8412 }, + { 8417, 8417 }, + { 8421, 8432 }, + { 11503, 11505 }, + { 11647, 11647 }, + { 11744, 11775 }, + { 12330, 12333 }, + { 12441, 12442 }, + { 42607, 42607 }, + { 42612, 42621 }, + { 42654, 42655 }, + { 42736, 42737 }, + { 43010, 43010 }, + { 43014, 43014 }, + { 43019, 43019 }, + { 43045, 43046 }, + { 43204, 43205 }, + { 43232, 43249 }, + { 43263, 43263 }, + { 43302, 43309 }, + { 43335, 43345 }, + { 43392, 43394 }, + { 43443, 43443 }, + { 43446, 43449 }, + { 43452, 43453 }, + { 43493, 43493 }, + { 43561, 43566 }, + { 43569, 43570 }, + { 43573, 43574 }, + { 43587, 43587 }, + { 43596, 43596 }, + { 43644, 43644 }, + { 43696, 43696 }, + { 43698, 43700 }, + { 43703, 43704 }, + { 43710, 43711 }, + { 43713, 43713 }, + { 43756, 43757 }, + { 43766, 43766 }, + { 44005, 44005 }, + { 44008, 44008 }, + { 44013, 44013 }, + { 64286, 64286 }, + { 65024, 65039 }, + { 65056, 65071 }, +}; +static const URange32 Mn_range32[] = { + { 66045, 66045 }, + { 66272, 66272 }, + { 66422, 66426 }, + { 68097, 68099 }, + { 68101, 68102 }, + { 68108, 68111 }, + { 68152, 68154 }, + { 68159, 68159 }, + { 68325, 68326 }, + { 68900, 68903 }, + { 69446, 69456 }, + { 69633, 69633 }, + { 69688, 69702 }, + { 69759, 69761 }, + { 69811, 69814 }, + { 69817, 69818 }, + { 69888, 69890 }, + { 69927, 69931 }, + { 69933, 69940 }, + { 70003, 70003 }, + { 70016, 70017 }, + { 70070, 70078 }, + { 70089, 70092 }, + { 70191, 70193 }, + { 70196, 70196 }, + { 70198, 70199 }, + { 70206, 70206 }, + { 70367, 70367 }, + { 70371, 70378 }, + { 70400, 70401 }, + { 70459, 70460 }, + { 70464, 70464 }, + { 70502, 70508 }, + { 70512, 70516 }, + { 70712, 70719 }, + { 70722, 70724 }, + { 70726, 70726 }, + { 70750, 70750 }, + { 70835, 70840 }, + { 70842, 70842 }, + { 70847, 70848 }, + { 70850, 70851 }, + { 71090, 71093 }, + { 71100, 71101 }, + { 71103, 71104 }, + { 71132, 71133 }, + { 71219, 71226 }, + { 71229, 71229 }, + { 71231, 71232 }, + { 71339, 71339 }, + { 71341, 71341 }, + { 71344, 71349 }, + { 71351, 71351 }, + { 71453, 71455 }, + { 71458, 71461 }, + { 71463, 71467 }, + { 71727, 71735 }, + { 71737, 71738 }, + { 72148, 72151 }, + { 72154, 72155 }, + { 72160, 72160 }, + { 72193, 72202 }, + { 72243, 72248 }, + { 72251, 72254 }, + { 72263, 72263 }, + { 72273, 72278 }, + { 72281, 72283 }, + { 72330, 72342 }, + { 72344, 72345 }, + { 72752, 72758 }, + { 72760, 72765 }, + { 72767, 72767 }, + { 72850, 72871 }, + { 72874, 72880 }, + { 72882, 72883 }, + { 72885, 72886 }, + { 73009, 73014 }, + { 73018, 73018 }, + { 73020, 73021 }, + { 73023, 73029 }, + { 73031, 73031 }, + { 73104, 73105 }, + { 73109, 73109 }, + { 73111, 73111 }, + { 73459, 73460 }, + { 92912, 92916 }, + { 92976, 92982 }, + { 94031, 94031 }, + { 94095, 94098 }, + { 113821, 113822 }, + { 119143, 119145 }, + { 119163, 119170 }, + { 119173, 119179 }, + { 119210, 119213 }, + { 119362, 119364 }, + { 121344, 121398 }, + { 121403, 121452 }, + { 121461, 121461 }, + { 121476, 121476 }, + { 121499, 121503 }, + { 121505, 121519 }, + { 122880, 122886 }, + { 122888, 122904 }, + { 122907, 122913 }, + { 122915, 122916 }, + { 122918, 122922 }, + { 123184, 123190 }, + { 123628, 123631 }, + { 125136, 125142 }, + { 125252, 125258 }, + { 917760, 917999 }, +}; +static const URange16 N_range16[] = { + { 48, 57 }, + { 178, 179 }, + { 185, 185 }, + { 188, 190 }, + { 1632, 1641 }, + { 1776, 1785 }, + { 1984, 1993 }, + { 2406, 2415 }, + { 2534, 2543 }, + { 2548, 2553 }, + { 2662, 2671 }, + { 2790, 2799 }, + { 2918, 2927 }, + { 2930, 2935 }, + { 3046, 3058 }, + { 3174, 3183 }, + { 3192, 3198 }, + { 3302, 3311 }, + { 3416, 3422 }, + { 3430, 3448 }, + { 3558, 3567 }, + { 3664, 3673 }, + { 3792, 3801 }, + { 3872, 3891 }, + { 4160, 4169 }, + { 4240, 4249 }, + { 4969, 4988 }, + { 5870, 5872 }, + { 6112, 6121 }, + { 6128, 6137 }, + { 6160, 6169 }, + { 6470, 6479 }, + { 6608, 6618 }, + { 6784, 6793 }, + { 6800, 6809 }, + { 6992, 7001 }, + { 7088, 7097 }, + { 7232, 7241 }, + { 7248, 7257 }, + { 8304, 8304 }, + { 8308, 8313 }, + { 8320, 8329 }, + { 8528, 8578 }, + { 8581, 8585 }, + { 9312, 9371 }, + { 9450, 9471 }, + { 10102, 10131 }, + { 11517, 11517 }, + { 12295, 12295 }, + { 12321, 12329 }, + { 12344, 12346 }, + { 12690, 12693 }, + { 12832, 12841 }, + { 12872, 12879 }, + { 12881, 12895 }, + { 12928, 12937 }, + { 12977, 12991 }, + { 42528, 42537 }, + { 42726, 42735 }, + { 43056, 43061 }, + { 43216, 43225 }, + { 43264, 43273 }, + { 43472, 43481 }, + { 43504, 43513 }, + { 43600, 43609 }, + { 44016, 44025 }, + { 65296, 65305 }, +}; +static const URange32 N_range32[] = { + { 65799, 65843 }, + { 65856, 65912 }, + { 65930, 65931 }, + { 66273, 66299 }, + { 66336, 66339 }, + { 66369, 66369 }, + { 66378, 66378 }, + { 66513, 66517 }, + { 66720, 66729 }, + { 67672, 67679 }, + { 67705, 67711 }, + { 67751, 67759 }, + { 67835, 67839 }, + { 67862, 67867 }, + { 68028, 68029 }, + { 68032, 68047 }, + { 68050, 68095 }, + { 68160, 68168 }, + { 68221, 68222 }, + { 68253, 68255 }, + { 68331, 68335 }, + { 68440, 68447 }, + { 68472, 68479 }, + { 68521, 68527 }, + { 68858, 68863 }, + { 68912, 68921 }, + { 69216, 69246 }, + { 69405, 69414 }, + { 69457, 69460 }, + { 69714, 69743 }, + { 69872, 69881 }, + { 69942, 69951 }, + { 70096, 70105 }, + { 70113, 70132 }, + { 70384, 70393 }, + { 70736, 70745 }, + { 70864, 70873 }, + { 71248, 71257 }, + { 71360, 71369 }, + { 71472, 71483 }, + { 71904, 71922 }, + { 72784, 72812 }, + { 73040, 73049 }, + { 73120, 73129 }, + { 73664, 73684 }, + { 74752, 74862 }, + { 92768, 92777 }, + { 93008, 93017 }, + { 93019, 93025 }, + { 93824, 93846 }, + { 119520, 119539 }, + { 119648, 119672 }, + { 120782, 120831 }, + { 123200, 123209 }, + { 123632, 123641 }, + { 125127, 125135 }, + { 125264, 125273 }, + { 126065, 126123 }, + { 126125, 126127 }, + { 126129, 126132 }, + { 126209, 126253 }, + { 126255, 126269 }, + { 127232, 127244 }, +}; +static const URange16 Nd_range16[] = { + { 48, 57 }, + { 1632, 1641 }, + { 1776, 1785 }, + { 1984, 1993 }, + { 2406, 2415 }, + { 2534, 2543 }, + { 2662, 2671 }, + { 2790, 2799 }, + { 2918, 2927 }, + { 3046, 3055 }, + { 3174, 3183 }, + { 3302, 3311 }, + { 3430, 3439 }, + { 3558, 3567 }, + { 3664, 3673 }, + { 3792, 3801 }, + { 3872, 3881 }, + { 4160, 4169 }, + { 4240, 4249 }, + { 6112, 6121 }, + { 6160, 6169 }, + { 6470, 6479 }, + { 6608, 6617 }, + { 6784, 6793 }, + { 6800, 6809 }, + { 6992, 7001 }, + { 7088, 7097 }, + { 7232, 7241 }, + { 7248, 7257 }, + { 42528, 42537 }, + { 43216, 43225 }, + { 43264, 43273 }, + { 43472, 43481 }, + { 43504, 43513 }, + { 43600, 43609 }, + { 44016, 44025 }, + { 65296, 65305 }, +}; +static const URange32 Nd_range32[] = { + { 66720, 66729 }, + { 68912, 68921 }, + { 69734, 69743 }, + { 69872, 69881 }, + { 69942, 69951 }, + { 70096, 70105 }, + { 70384, 70393 }, + { 70736, 70745 }, + { 70864, 70873 }, + { 71248, 71257 }, + { 71360, 71369 }, + { 71472, 71481 }, + { 71904, 71913 }, + { 72784, 72793 }, + { 73040, 73049 }, + { 73120, 73129 }, + { 92768, 92777 }, + { 93008, 93017 }, + { 120782, 120831 }, + { 123200, 123209 }, + { 123632, 123641 }, + { 125264, 125273 }, +}; +static const URange16 Nl_range16[] = { + { 5870, 5872 }, + { 8544, 8578 }, + { 8581, 8584 }, + { 12295, 12295 }, + { 12321, 12329 }, + { 12344, 12346 }, + { 42726, 42735 }, +}; +static const URange32 Nl_range32[] = { + { 65856, 65908 }, + { 66369, 66369 }, + { 66378, 66378 }, + { 66513, 66517 }, + { 74752, 74862 }, +}; +static const URange16 No_range16[] = { + { 178, 179 }, + { 185, 185 }, + { 188, 190 }, + { 2548, 2553 }, + { 2930, 2935 }, + { 3056, 3058 }, + { 3192, 3198 }, + { 3416, 3422 }, + { 3440, 3448 }, + { 3882, 3891 }, + { 4969, 4988 }, + { 6128, 6137 }, + { 6618, 6618 }, + { 8304, 8304 }, + { 8308, 8313 }, + { 8320, 8329 }, + { 8528, 8543 }, + { 8585, 8585 }, + { 9312, 9371 }, + { 9450, 9471 }, + { 10102, 10131 }, + { 11517, 11517 }, + { 12690, 12693 }, + { 12832, 12841 }, + { 12872, 12879 }, + { 12881, 12895 }, + { 12928, 12937 }, + { 12977, 12991 }, + { 43056, 43061 }, +}; +static const URange32 No_range32[] = { + { 65799, 65843 }, + { 65909, 65912 }, + { 65930, 65931 }, + { 66273, 66299 }, + { 66336, 66339 }, + { 67672, 67679 }, + { 67705, 67711 }, + { 67751, 67759 }, + { 67835, 67839 }, + { 67862, 67867 }, + { 68028, 68029 }, + { 68032, 68047 }, + { 68050, 68095 }, + { 68160, 68168 }, + { 68221, 68222 }, + { 68253, 68255 }, + { 68331, 68335 }, + { 68440, 68447 }, + { 68472, 68479 }, + { 68521, 68527 }, + { 68858, 68863 }, + { 69216, 69246 }, + { 69405, 69414 }, + { 69457, 69460 }, + { 69714, 69733 }, + { 70113, 70132 }, + { 71482, 71483 }, + { 71914, 71922 }, + { 72794, 72812 }, + { 73664, 73684 }, + { 93019, 93025 }, + { 93824, 93846 }, + { 119520, 119539 }, + { 119648, 119672 }, + { 125127, 125135 }, + { 126065, 126123 }, + { 126125, 126127 }, + { 126129, 126132 }, + { 126209, 126253 }, + { 126255, 126269 }, + { 127232, 127244 }, +}; +static const URange16 P_range16[] = { + { 33, 35 }, + { 37, 42 }, + { 44, 47 }, + { 58, 59 }, + { 63, 64 }, + { 91, 93 }, + { 95, 95 }, + { 123, 123 }, + { 125, 125 }, + { 161, 161 }, + { 167, 167 }, + { 171, 171 }, + { 182, 183 }, + { 187, 187 }, + { 191, 191 }, + { 894, 894 }, + { 903, 903 }, + { 1370, 1375 }, + { 1417, 1418 }, + { 1470, 1470 }, + { 1472, 1472 }, + { 1475, 1475 }, + { 1478, 1478 }, + { 1523, 1524 }, + { 1545, 1546 }, + { 1548, 1549 }, + { 1563, 1563 }, + { 1566, 1567 }, + { 1642, 1645 }, + { 1748, 1748 }, + { 1792, 1805 }, + { 2039, 2041 }, + { 2096, 2110 }, + { 2142, 2142 }, + { 2404, 2405 }, + { 2416, 2416 }, + { 2557, 2557 }, + { 2678, 2678 }, + { 2800, 2800 }, + { 3191, 3191 }, + { 3204, 3204 }, + { 3572, 3572 }, + { 3663, 3663 }, + { 3674, 3675 }, + { 3844, 3858 }, + { 3860, 3860 }, + { 3898, 3901 }, + { 3973, 3973 }, + { 4048, 4052 }, + { 4057, 4058 }, + { 4170, 4175 }, + { 4347, 4347 }, + { 4960, 4968 }, + { 5120, 5120 }, + { 5742, 5742 }, + { 5787, 5788 }, + { 5867, 5869 }, + { 5941, 5942 }, + { 6100, 6102 }, + { 6104, 6106 }, + { 6144, 6154 }, + { 6468, 6469 }, + { 6686, 6687 }, + { 6816, 6822 }, + { 6824, 6829 }, + { 7002, 7008 }, + { 7164, 7167 }, + { 7227, 7231 }, + { 7294, 7295 }, + { 7360, 7367 }, + { 7379, 7379 }, + { 8208, 8231 }, + { 8240, 8259 }, + { 8261, 8273 }, + { 8275, 8286 }, + { 8317, 8318 }, + { 8333, 8334 }, + { 8968, 8971 }, + { 9001, 9002 }, + { 10088, 10101 }, + { 10181, 10182 }, + { 10214, 10223 }, + { 10627, 10648 }, + { 10712, 10715 }, + { 10748, 10749 }, + { 11513, 11516 }, + { 11518, 11519 }, + { 11632, 11632 }, + { 11776, 11822 }, + { 11824, 11855 }, + { 12289, 12291 }, + { 12296, 12305 }, + { 12308, 12319 }, + { 12336, 12336 }, + { 12349, 12349 }, + { 12448, 12448 }, + { 12539, 12539 }, + { 42238, 42239 }, + { 42509, 42511 }, + { 42611, 42611 }, + { 42622, 42622 }, + { 42738, 42743 }, + { 43124, 43127 }, + { 43214, 43215 }, + { 43256, 43258 }, + { 43260, 43260 }, + { 43310, 43311 }, + { 43359, 43359 }, + { 43457, 43469 }, + { 43486, 43487 }, + { 43612, 43615 }, + { 43742, 43743 }, + { 43760, 43761 }, + { 44011, 44011 }, + { 64830, 64831 }, + { 65040, 65049 }, + { 65072, 65106 }, + { 65108, 65121 }, + { 65123, 65123 }, + { 65128, 65128 }, + { 65130, 65131 }, + { 65281, 65283 }, + { 65285, 65290 }, + { 65292, 65295 }, + { 65306, 65307 }, + { 65311, 65312 }, + { 65339, 65341 }, + { 65343, 65343 }, + { 65371, 65371 }, + { 65373, 65373 }, + { 65375, 65381 }, +}; +static const URange32 P_range32[] = { + { 65792, 65794 }, + { 66463, 66463 }, + { 66512, 66512 }, + { 66927, 66927 }, + { 67671, 67671 }, + { 67871, 67871 }, + { 67903, 67903 }, + { 68176, 68184 }, + { 68223, 68223 }, + { 68336, 68342 }, + { 68409, 68415 }, + { 68505, 68508 }, + { 69461, 69465 }, + { 69703, 69709 }, + { 69819, 69820 }, + { 69822, 69825 }, + { 69952, 69955 }, + { 70004, 70005 }, + { 70085, 70088 }, + { 70093, 70093 }, + { 70107, 70107 }, + { 70109, 70111 }, + { 70200, 70205 }, + { 70313, 70313 }, + { 70731, 70735 }, + { 70747, 70747 }, + { 70749, 70749 }, + { 70854, 70854 }, + { 71105, 71127 }, + { 71233, 71235 }, + { 71264, 71276 }, + { 71484, 71486 }, + { 71739, 71739 }, + { 72162, 72162 }, + { 72255, 72262 }, + { 72346, 72348 }, + { 72350, 72354 }, + { 72769, 72773 }, + { 72816, 72817 }, + { 73463, 73464 }, + { 73727, 73727 }, + { 74864, 74868 }, + { 92782, 92783 }, + { 92917, 92917 }, + { 92983, 92987 }, + { 92996, 92996 }, + { 93847, 93850 }, + { 94178, 94178 }, + { 113823, 113823 }, + { 121479, 121483 }, + { 125278, 125279 }, +}; +static const URange16 Pc_range16[] = { + { 95, 95 }, + { 8255, 8256 }, + { 8276, 8276 }, + { 65075, 65076 }, + { 65101, 65103 }, + { 65343, 65343 }, +}; +static const URange16 Pd_range16[] = { + { 45, 45 }, + { 1418, 1418 }, + { 1470, 1470 }, + { 5120, 5120 }, + { 6150, 6150 }, + { 8208, 8213 }, + { 11799, 11799 }, + { 11802, 11802 }, + { 11834, 11835 }, + { 11840, 11840 }, + { 12316, 12316 }, + { 12336, 12336 }, + { 12448, 12448 }, + { 65073, 65074 }, + { 65112, 65112 }, + { 65123, 65123 }, + { 65293, 65293 }, +}; +static const URange16 Pe_range16[] = { + { 41, 41 }, + { 93, 93 }, + { 125, 125 }, + { 3899, 3899 }, + { 3901, 3901 }, + { 5788, 5788 }, + { 8262, 8262 }, + { 8318, 8318 }, + { 8334, 8334 }, + { 8969, 8969 }, + { 8971, 8971 }, + { 9002, 9002 }, + { 10089, 10089 }, + { 10091, 10091 }, + { 10093, 10093 }, + { 10095, 10095 }, + { 10097, 10097 }, + { 10099, 10099 }, + { 10101, 10101 }, + { 10182, 10182 }, + { 10215, 10215 }, + { 10217, 10217 }, + { 10219, 10219 }, + { 10221, 10221 }, + { 10223, 10223 }, + { 10628, 10628 }, + { 10630, 10630 }, + { 10632, 10632 }, + { 10634, 10634 }, + { 10636, 10636 }, + { 10638, 10638 }, + { 10640, 10640 }, + { 10642, 10642 }, + { 10644, 10644 }, + { 10646, 10646 }, + { 10648, 10648 }, + { 10713, 10713 }, + { 10715, 10715 }, + { 10749, 10749 }, + { 11811, 11811 }, + { 11813, 11813 }, + { 11815, 11815 }, + { 11817, 11817 }, + { 12297, 12297 }, + { 12299, 12299 }, + { 12301, 12301 }, + { 12303, 12303 }, + { 12305, 12305 }, + { 12309, 12309 }, + { 12311, 12311 }, + { 12313, 12313 }, + { 12315, 12315 }, + { 12318, 12319 }, + { 64830, 64830 }, + { 65048, 65048 }, + { 65078, 65078 }, + { 65080, 65080 }, + { 65082, 65082 }, + { 65084, 65084 }, + { 65086, 65086 }, + { 65088, 65088 }, + { 65090, 65090 }, + { 65092, 65092 }, + { 65096, 65096 }, + { 65114, 65114 }, + { 65116, 65116 }, + { 65118, 65118 }, + { 65289, 65289 }, + { 65341, 65341 }, + { 65373, 65373 }, + { 65376, 65376 }, + { 65379, 65379 }, +}; +static const URange16 Pf_range16[] = { + { 187, 187 }, + { 8217, 8217 }, + { 8221, 8221 }, + { 8250, 8250 }, + { 11779, 11779 }, + { 11781, 11781 }, + { 11786, 11786 }, + { 11789, 11789 }, + { 11805, 11805 }, + { 11809, 11809 }, +}; +static const URange16 Pi_range16[] = { + { 171, 171 }, + { 8216, 8216 }, + { 8219, 8220 }, + { 8223, 8223 }, + { 8249, 8249 }, + { 11778, 11778 }, + { 11780, 11780 }, + { 11785, 11785 }, + { 11788, 11788 }, + { 11804, 11804 }, + { 11808, 11808 }, +}; +static const URange16 Po_range16[] = { + { 33, 35 }, + { 37, 39 }, + { 42, 42 }, + { 44, 44 }, + { 46, 47 }, + { 58, 59 }, + { 63, 64 }, + { 92, 92 }, + { 161, 161 }, + { 167, 167 }, + { 182, 183 }, + { 191, 191 }, + { 894, 894 }, + { 903, 903 }, + { 1370, 1375 }, + { 1417, 1417 }, + { 1472, 1472 }, + { 1475, 1475 }, + { 1478, 1478 }, + { 1523, 1524 }, + { 1545, 1546 }, + { 1548, 1549 }, + { 1563, 1563 }, + { 1566, 1567 }, + { 1642, 1645 }, + { 1748, 1748 }, + { 1792, 1805 }, + { 2039, 2041 }, + { 2096, 2110 }, + { 2142, 2142 }, + { 2404, 2405 }, + { 2416, 2416 }, + { 2557, 2557 }, + { 2678, 2678 }, + { 2800, 2800 }, + { 3191, 3191 }, + { 3204, 3204 }, + { 3572, 3572 }, + { 3663, 3663 }, + { 3674, 3675 }, + { 3844, 3858 }, + { 3860, 3860 }, + { 3973, 3973 }, + { 4048, 4052 }, + { 4057, 4058 }, + { 4170, 4175 }, + { 4347, 4347 }, + { 4960, 4968 }, + { 5742, 5742 }, + { 5867, 5869 }, + { 5941, 5942 }, + { 6100, 6102 }, + { 6104, 6106 }, + { 6144, 6149 }, + { 6151, 6154 }, + { 6468, 6469 }, + { 6686, 6687 }, + { 6816, 6822 }, + { 6824, 6829 }, + { 7002, 7008 }, + { 7164, 7167 }, + { 7227, 7231 }, + { 7294, 7295 }, + { 7360, 7367 }, + { 7379, 7379 }, + { 8214, 8215 }, + { 8224, 8231 }, + { 8240, 8248 }, + { 8251, 8254 }, + { 8257, 8259 }, + { 8263, 8273 }, + { 8275, 8275 }, + { 8277, 8286 }, + { 11513, 11516 }, + { 11518, 11519 }, + { 11632, 11632 }, + { 11776, 11777 }, + { 11782, 11784 }, + { 11787, 11787 }, + { 11790, 11798 }, + { 11800, 11801 }, + { 11803, 11803 }, + { 11806, 11807 }, + { 11818, 11822 }, + { 11824, 11833 }, + { 11836, 11839 }, + { 11841, 11841 }, + { 11843, 11855 }, + { 12289, 12291 }, + { 12349, 12349 }, + { 12539, 12539 }, + { 42238, 42239 }, + { 42509, 42511 }, + { 42611, 42611 }, + { 42622, 42622 }, + { 42738, 42743 }, + { 43124, 43127 }, + { 43214, 43215 }, + { 43256, 43258 }, + { 43260, 43260 }, + { 43310, 43311 }, + { 43359, 43359 }, + { 43457, 43469 }, + { 43486, 43487 }, + { 43612, 43615 }, + { 43742, 43743 }, + { 43760, 43761 }, + { 44011, 44011 }, + { 65040, 65046 }, + { 65049, 65049 }, + { 65072, 65072 }, + { 65093, 65094 }, + { 65097, 65100 }, + { 65104, 65106 }, + { 65108, 65111 }, + { 65119, 65121 }, + { 65128, 65128 }, + { 65130, 65131 }, + { 65281, 65283 }, + { 65285, 65287 }, + { 65290, 65290 }, + { 65292, 65292 }, + { 65294, 65295 }, + { 65306, 65307 }, + { 65311, 65312 }, + { 65340, 65340 }, + { 65377, 65377 }, + { 65380, 65381 }, +}; +static const URange32 Po_range32[] = { + { 65792, 65794 }, + { 66463, 66463 }, + { 66512, 66512 }, + { 66927, 66927 }, + { 67671, 67671 }, + { 67871, 67871 }, + { 67903, 67903 }, + { 68176, 68184 }, + { 68223, 68223 }, + { 68336, 68342 }, + { 68409, 68415 }, + { 68505, 68508 }, + { 69461, 69465 }, + { 69703, 69709 }, + { 69819, 69820 }, + { 69822, 69825 }, + { 69952, 69955 }, + { 70004, 70005 }, + { 70085, 70088 }, + { 70093, 70093 }, + { 70107, 70107 }, + { 70109, 70111 }, + { 70200, 70205 }, + { 70313, 70313 }, + { 70731, 70735 }, + { 70747, 70747 }, + { 70749, 70749 }, + { 70854, 70854 }, + { 71105, 71127 }, + { 71233, 71235 }, + { 71264, 71276 }, + { 71484, 71486 }, + { 71739, 71739 }, + { 72162, 72162 }, + { 72255, 72262 }, + { 72346, 72348 }, + { 72350, 72354 }, + { 72769, 72773 }, + { 72816, 72817 }, + { 73463, 73464 }, + { 73727, 73727 }, + { 74864, 74868 }, + { 92782, 92783 }, + { 92917, 92917 }, + { 92983, 92987 }, + { 92996, 92996 }, + { 93847, 93850 }, + { 94178, 94178 }, + { 113823, 113823 }, + { 121479, 121483 }, + { 125278, 125279 }, +}; +static const URange16 Ps_range16[] = { + { 40, 40 }, + { 91, 91 }, + { 123, 123 }, + { 3898, 3898 }, + { 3900, 3900 }, + { 5787, 5787 }, + { 8218, 8218 }, + { 8222, 8222 }, + { 8261, 8261 }, + { 8317, 8317 }, + { 8333, 8333 }, + { 8968, 8968 }, + { 8970, 8970 }, + { 9001, 9001 }, + { 10088, 10088 }, + { 10090, 10090 }, + { 10092, 10092 }, + { 10094, 10094 }, + { 10096, 10096 }, + { 10098, 10098 }, + { 10100, 10100 }, + { 10181, 10181 }, + { 10214, 10214 }, + { 10216, 10216 }, + { 10218, 10218 }, + { 10220, 10220 }, + { 10222, 10222 }, + { 10627, 10627 }, + { 10629, 10629 }, + { 10631, 10631 }, + { 10633, 10633 }, + { 10635, 10635 }, + { 10637, 10637 }, + { 10639, 10639 }, + { 10641, 10641 }, + { 10643, 10643 }, + { 10645, 10645 }, + { 10647, 10647 }, + { 10712, 10712 }, + { 10714, 10714 }, + { 10748, 10748 }, + { 11810, 11810 }, + { 11812, 11812 }, + { 11814, 11814 }, + { 11816, 11816 }, + { 11842, 11842 }, + { 12296, 12296 }, + { 12298, 12298 }, + { 12300, 12300 }, + { 12302, 12302 }, + { 12304, 12304 }, + { 12308, 12308 }, + { 12310, 12310 }, + { 12312, 12312 }, + { 12314, 12314 }, + { 12317, 12317 }, + { 64831, 64831 }, + { 65047, 65047 }, + { 65077, 65077 }, + { 65079, 65079 }, + { 65081, 65081 }, + { 65083, 65083 }, + { 65085, 65085 }, + { 65087, 65087 }, + { 65089, 65089 }, + { 65091, 65091 }, + { 65095, 65095 }, + { 65113, 65113 }, + { 65115, 65115 }, + { 65117, 65117 }, + { 65288, 65288 }, + { 65339, 65339 }, + { 65371, 65371 }, + { 65375, 65375 }, + { 65378, 65378 }, +}; +static const URange16 S_range16[] = { + { 36, 36 }, + { 43, 43 }, + { 60, 62 }, + { 94, 94 }, + { 96, 96 }, + { 124, 124 }, + { 126, 126 }, + { 162, 166 }, + { 168, 169 }, + { 172, 172 }, + { 174, 177 }, + { 180, 180 }, + { 184, 184 }, + { 215, 215 }, + { 247, 247 }, + { 706, 709 }, + { 722, 735 }, + { 741, 747 }, + { 749, 749 }, + { 751, 767 }, + { 885, 885 }, + { 900, 901 }, + { 1014, 1014 }, + { 1154, 1154 }, + { 1421, 1423 }, + { 1542, 1544 }, + { 1547, 1547 }, + { 1550, 1551 }, + { 1758, 1758 }, + { 1769, 1769 }, + { 1789, 1790 }, + { 2038, 2038 }, + { 2046, 2047 }, + { 2546, 2547 }, + { 2554, 2555 }, + { 2801, 2801 }, + { 2928, 2928 }, + { 3059, 3066 }, + { 3199, 3199 }, + { 3407, 3407 }, + { 3449, 3449 }, + { 3647, 3647 }, + { 3841, 3843 }, + { 3859, 3859 }, + { 3861, 3863 }, + { 3866, 3871 }, + { 3892, 3892 }, + { 3894, 3894 }, + { 3896, 3896 }, + { 4030, 4037 }, + { 4039, 4044 }, + { 4046, 4047 }, + { 4053, 4056 }, + { 4254, 4255 }, + { 5008, 5017 }, + { 5741, 5741 }, + { 6107, 6107 }, + { 6464, 6464 }, + { 6622, 6655 }, + { 7009, 7018 }, + { 7028, 7036 }, + { 8125, 8125 }, + { 8127, 8129 }, + { 8141, 8143 }, + { 8157, 8159 }, + { 8173, 8175 }, + { 8189, 8190 }, + { 8260, 8260 }, + { 8274, 8274 }, + { 8314, 8316 }, + { 8330, 8332 }, + { 8352, 8383 }, + { 8448, 8449 }, + { 8451, 8454 }, + { 8456, 8457 }, + { 8468, 8468 }, + { 8470, 8472 }, + { 8478, 8483 }, + { 8485, 8485 }, + { 8487, 8487 }, + { 8489, 8489 }, + { 8494, 8494 }, + { 8506, 8507 }, + { 8512, 8516 }, + { 8522, 8525 }, + { 8527, 8527 }, + { 8586, 8587 }, + { 8592, 8967 }, + { 8972, 9000 }, + { 9003, 9254 }, + { 9280, 9290 }, + { 9372, 9449 }, + { 9472, 10087 }, + { 10132, 10180 }, + { 10183, 10213 }, + { 10224, 10626 }, + { 10649, 10711 }, + { 10716, 10747 }, + { 10750, 11123 }, + { 11126, 11157 }, + { 11160, 11263 }, + { 11493, 11498 }, + { 11904, 11929 }, + { 11931, 12019 }, + { 12032, 12245 }, + { 12272, 12283 }, + { 12292, 12292 }, + { 12306, 12307 }, + { 12320, 12320 }, + { 12342, 12343 }, + { 12350, 12351 }, + { 12443, 12444 }, + { 12688, 12689 }, + { 12694, 12703 }, + { 12736, 12771 }, + { 12800, 12830 }, + { 12842, 12871 }, + { 12880, 12880 }, + { 12896, 12927 }, + { 12938, 12976 }, + { 12992, 13311 }, + { 19904, 19967 }, + { 42128, 42182 }, + { 42752, 42774 }, + { 42784, 42785 }, + { 42889, 42890 }, + { 43048, 43051 }, + { 43062, 43065 }, + { 43639, 43641 }, + { 43867, 43867 }, + { 64297, 64297 }, + { 64434, 64449 }, + { 65020, 65021 }, + { 65122, 65122 }, + { 65124, 65126 }, + { 65129, 65129 }, + { 65284, 65284 }, + { 65291, 65291 }, + { 65308, 65310 }, + { 65342, 65342 }, + { 65344, 65344 }, + { 65372, 65372 }, + { 65374, 65374 }, + { 65504, 65510 }, + { 65512, 65518 }, + { 65532, 65533 }, +}; +static const URange32 S_range32[] = { + { 65847, 65855 }, + { 65913, 65929 }, + { 65932, 65934 }, + { 65936, 65947 }, + { 65952, 65952 }, + { 66000, 66044 }, + { 67703, 67704 }, + { 68296, 68296 }, + { 71487, 71487 }, + { 73685, 73713 }, + { 92988, 92991 }, + { 92997, 92997 }, + { 113820, 113820 }, + { 118784, 119029 }, + { 119040, 119078 }, + { 119081, 119140 }, + { 119146, 119148 }, + { 119171, 119172 }, + { 119180, 119209 }, + { 119214, 119272 }, + { 119296, 119361 }, + { 119365, 119365 }, + { 119552, 119638 }, + { 120513, 120513 }, + { 120539, 120539 }, + { 120571, 120571 }, + { 120597, 120597 }, + { 120629, 120629 }, + { 120655, 120655 }, + { 120687, 120687 }, + { 120713, 120713 }, + { 120745, 120745 }, + { 120771, 120771 }, + { 120832, 121343 }, + { 121399, 121402 }, + { 121453, 121460 }, + { 121462, 121475 }, + { 121477, 121478 }, + { 123215, 123215 }, + { 123647, 123647 }, + { 126124, 126124 }, + { 126128, 126128 }, + { 126254, 126254 }, + { 126704, 126705 }, + { 126976, 127019 }, + { 127024, 127123 }, + { 127136, 127150 }, + { 127153, 127167 }, + { 127169, 127183 }, + { 127185, 127221 }, + { 127248, 127340 }, + { 127344, 127404 }, + { 127462, 127490 }, + { 127504, 127547 }, + { 127552, 127560 }, + { 127568, 127569 }, + { 127584, 127589 }, + { 127744, 128725 }, + { 128736, 128748 }, + { 128752, 128762 }, + { 128768, 128883 }, + { 128896, 128984 }, + { 128992, 129003 }, + { 129024, 129035 }, + { 129040, 129095 }, + { 129104, 129113 }, + { 129120, 129159 }, + { 129168, 129197 }, + { 129280, 129291 }, + { 129293, 129393 }, + { 129395, 129398 }, + { 129402, 129442 }, + { 129445, 129450 }, + { 129454, 129482 }, + { 129485, 129619 }, + { 129632, 129645 }, + { 129648, 129651 }, + { 129656, 129658 }, + { 129664, 129666 }, + { 129680, 129685 }, +}; +static const URange16 Sc_range16[] = { + { 36, 36 }, + { 162, 165 }, + { 1423, 1423 }, + { 1547, 1547 }, + { 2046, 2047 }, + { 2546, 2547 }, + { 2555, 2555 }, + { 2801, 2801 }, + { 3065, 3065 }, + { 3647, 3647 }, + { 6107, 6107 }, + { 8352, 8383 }, + { 43064, 43064 }, + { 65020, 65020 }, + { 65129, 65129 }, + { 65284, 65284 }, + { 65504, 65505 }, + { 65509, 65510 }, +}; +static const URange32 Sc_range32[] = { + { 73693, 73696 }, + { 123647, 123647 }, + { 126128, 126128 }, +}; +static const URange16 Sk_range16[] = { + { 94, 94 }, + { 96, 96 }, + { 168, 168 }, + { 175, 175 }, + { 180, 180 }, + { 184, 184 }, + { 706, 709 }, + { 722, 735 }, + { 741, 747 }, + { 749, 749 }, + { 751, 767 }, + { 885, 885 }, + { 900, 901 }, + { 8125, 8125 }, + { 8127, 8129 }, + { 8141, 8143 }, + { 8157, 8159 }, + { 8173, 8175 }, + { 8189, 8190 }, + { 12443, 12444 }, + { 42752, 42774 }, + { 42784, 42785 }, + { 42889, 42890 }, + { 43867, 43867 }, + { 64434, 64449 }, + { 65342, 65342 }, + { 65344, 65344 }, + { 65507, 65507 }, +}; +static const URange32 Sk_range32[] = { + { 127995, 127999 }, +}; +static const URange16 Sm_range16[] = { + { 43, 43 }, + { 60, 62 }, + { 124, 124 }, + { 126, 126 }, + { 172, 172 }, + { 177, 177 }, + { 215, 215 }, + { 247, 247 }, + { 1014, 1014 }, + { 1542, 1544 }, + { 8260, 8260 }, + { 8274, 8274 }, + { 8314, 8316 }, + { 8330, 8332 }, + { 8472, 8472 }, + { 8512, 8516 }, + { 8523, 8523 }, + { 8592, 8596 }, + { 8602, 8603 }, + { 8608, 8608 }, + { 8611, 8611 }, + { 8614, 8614 }, + { 8622, 8622 }, + { 8654, 8655 }, + { 8658, 8658 }, + { 8660, 8660 }, + { 8692, 8959 }, + { 8992, 8993 }, + { 9084, 9084 }, + { 9115, 9139 }, + { 9180, 9185 }, + { 9655, 9655 }, + { 9665, 9665 }, + { 9720, 9727 }, + { 9839, 9839 }, + { 10176, 10180 }, + { 10183, 10213 }, + { 10224, 10239 }, + { 10496, 10626 }, + { 10649, 10711 }, + { 10716, 10747 }, + { 10750, 11007 }, + { 11056, 11076 }, + { 11079, 11084 }, + { 64297, 64297 }, + { 65122, 65122 }, + { 65124, 65126 }, + { 65291, 65291 }, + { 65308, 65310 }, + { 65372, 65372 }, + { 65374, 65374 }, + { 65506, 65506 }, + { 65513, 65516 }, +}; +static const URange32 Sm_range32[] = { + { 120513, 120513 }, + { 120539, 120539 }, + { 120571, 120571 }, + { 120597, 120597 }, + { 120629, 120629 }, + { 120655, 120655 }, + { 120687, 120687 }, + { 120713, 120713 }, + { 120745, 120745 }, + { 120771, 120771 }, + { 126704, 126705 }, +}; +static const URange16 So_range16[] = { + { 166, 166 }, + { 169, 169 }, + { 174, 174 }, + { 176, 176 }, + { 1154, 1154 }, + { 1421, 1422 }, + { 1550, 1551 }, + { 1758, 1758 }, + { 1769, 1769 }, + { 1789, 1790 }, + { 2038, 2038 }, + { 2554, 2554 }, + { 2928, 2928 }, + { 3059, 3064 }, + { 3066, 3066 }, + { 3199, 3199 }, + { 3407, 3407 }, + { 3449, 3449 }, + { 3841, 3843 }, + { 3859, 3859 }, + { 3861, 3863 }, + { 3866, 3871 }, + { 3892, 3892 }, + { 3894, 3894 }, + { 3896, 3896 }, + { 4030, 4037 }, + { 4039, 4044 }, + { 4046, 4047 }, + { 4053, 4056 }, + { 4254, 4255 }, + { 5008, 5017 }, + { 5741, 5741 }, + { 6464, 6464 }, + { 6622, 6655 }, + { 7009, 7018 }, + { 7028, 7036 }, + { 8448, 8449 }, + { 8451, 8454 }, + { 8456, 8457 }, + { 8468, 8468 }, + { 8470, 8471 }, + { 8478, 8483 }, + { 8485, 8485 }, + { 8487, 8487 }, + { 8489, 8489 }, + { 8494, 8494 }, + { 8506, 8507 }, + { 8522, 8522 }, + { 8524, 8525 }, + { 8527, 8527 }, + { 8586, 8587 }, + { 8597, 8601 }, + { 8604, 8607 }, + { 8609, 8610 }, + { 8612, 8613 }, + { 8615, 8621 }, + { 8623, 8653 }, + { 8656, 8657 }, + { 8659, 8659 }, + { 8661, 8691 }, + { 8960, 8967 }, + { 8972, 8991 }, + { 8994, 9000 }, + { 9003, 9083 }, + { 9085, 9114 }, + { 9140, 9179 }, + { 9186, 9254 }, + { 9280, 9290 }, + { 9372, 9449 }, + { 9472, 9654 }, + { 9656, 9664 }, + { 9666, 9719 }, + { 9728, 9838 }, + { 9840, 10087 }, + { 10132, 10175 }, + { 10240, 10495 }, + { 11008, 11055 }, + { 11077, 11078 }, + { 11085, 11123 }, + { 11126, 11157 }, + { 11160, 11263 }, + { 11493, 11498 }, + { 11904, 11929 }, + { 11931, 12019 }, + { 12032, 12245 }, + { 12272, 12283 }, + { 12292, 12292 }, + { 12306, 12307 }, + { 12320, 12320 }, + { 12342, 12343 }, + { 12350, 12351 }, + { 12688, 12689 }, + { 12694, 12703 }, + { 12736, 12771 }, + { 12800, 12830 }, + { 12842, 12871 }, + { 12880, 12880 }, + { 12896, 12927 }, + { 12938, 12976 }, + { 12992, 13311 }, + { 19904, 19967 }, + { 42128, 42182 }, + { 43048, 43051 }, + { 43062, 43063 }, + { 43065, 43065 }, + { 43639, 43641 }, + { 65021, 65021 }, + { 65508, 65508 }, + { 65512, 65512 }, + { 65517, 65518 }, + { 65532, 65533 }, +}; +static const URange32 So_range32[] = { + { 65847, 65855 }, + { 65913, 65929 }, + { 65932, 65934 }, + { 65936, 65947 }, + { 65952, 65952 }, + { 66000, 66044 }, + { 67703, 67704 }, + { 68296, 68296 }, + { 71487, 71487 }, + { 73685, 73692 }, + { 73697, 73713 }, + { 92988, 92991 }, + { 92997, 92997 }, + { 113820, 113820 }, + { 118784, 119029 }, + { 119040, 119078 }, + { 119081, 119140 }, + { 119146, 119148 }, + { 119171, 119172 }, + { 119180, 119209 }, + { 119214, 119272 }, + { 119296, 119361 }, + { 119365, 119365 }, + { 119552, 119638 }, + { 120832, 121343 }, + { 121399, 121402 }, + { 121453, 121460 }, + { 121462, 121475 }, + { 121477, 121478 }, + { 123215, 123215 }, + { 126124, 126124 }, + { 126254, 126254 }, + { 126976, 127019 }, + { 127024, 127123 }, + { 127136, 127150 }, + { 127153, 127167 }, + { 127169, 127183 }, + { 127185, 127221 }, + { 127248, 127340 }, + { 127344, 127404 }, + { 127462, 127490 }, + { 127504, 127547 }, + { 127552, 127560 }, + { 127568, 127569 }, + { 127584, 127589 }, + { 127744, 127994 }, + { 128000, 128725 }, + { 128736, 128748 }, + { 128752, 128762 }, + { 128768, 128883 }, + { 128896, 128984 }, + { 128992, 129003 }, + { 129024, 129035 }, + { 129040, 129095 }, + { 129104, 129113 }, + { 129120, 129159 }, + { 129168, 129197 }, + { 129280, 129291 }, + { 129293, 129393 }, + { 129395, 129398 }, + { 129402, 129442 }, + { 129445, 129450 }, + { 129454, 129482 }, + { 129485, 129619 }, + { 129632, 129645 }, + { 129648, 129651 }, + { 129656, 129658 }, + { 129664, 129666 }, + { 129680, 129685 }, +}; +static const URange16 Z_range16[] = { + { 32, 32 }, + { 160, 160 }, + { 5760, 5760 }, + { 8192, 8202 }, + { 8232, 8233 }, + { 8239, 8239 }, + { 8287, 8287 }, + { 12288, 12288 }, +}; +static const URange16 Zl_range16[] = { + { 8232, 8232 }, +}; +static const URange16 Zp_range16[] = { + { 8233, 8233 }, +}; +static const URange16 Zs_range16[] = { + { 32, 32 }, + { 160, 160 }, + { 5760, 5760 }, + { 8192, 8202 }, + { 8239, 8239 }, + { 8287, 8287 }, + { 12288, 12288 }, +}; +static const URange32 Adlam_range32[] = { + { 125184, 125259 }, + { 125264, 125273 }, + { 125278, 125279 }, +}; +static const URange32 Ahom_range32[] = { + { 71424, 71450 }, + { 71453, 71467 }, + { 71472, 71487 }, +}; +static const URange32 Anatolian_Hieroglyphs_range32[] = { + { 82944, 83526 }, +}; +static const URange16 Arabic_range16[] = { + { 1536, 1540 }, + { 1542, 1547 }, + { 1549, 1562 }, + { 1564, 1564 }, + { 1566, 1566 }, + { 1568, 1599 }, + { 1601, 1610 }, + { 1622, 1647 }, + { 1649, 1756 }, + { 1758, 1791 }, + { 1872, 1919 }, + { 2208, 2228 }, + { 2230, 2237 }, + { 2259, 2273 }, + { 2275, 2303 }, + { 64336, 64449 }, + { 64467, 64829 }, + { 64848, 64911 }, + { 64914, 64967 }, + { 65008, 65021 }, + { 65136, 65140 }, + { 65142, 65276 }, +}; +static const URange32 Arabic_range32[] = { + { 69216, 69246 }, + { 126464, 126467 }, + { 126469, 126495 }, + { 126497, 126498 }, + { 126500, 126500 }, + { 126503, 126503 }, + { 126505, 126514 }, + { 126516, 126519 }, + { 126521, 126521 }, + { 126523, 126523 }, + { 126530, 126530 }, + { 126535, 126535 }, + { 126537, 126537 }, + { 126539, 126539 }, + { 126541, 126543 }, + { 126545, 126546 }, + { 126548, 126548 }, + { 126551, 126551 }, + { 126553, 126553 }, + { 126555, 126555 }, + { 126557, 126557 }, + { 126559, 126559 }, + { 126561, 126562 }, + { 126564, 126564 }, + { 126567, 126570 }, + { 126572, 126578 }, + { 126580, 126583 }, + { 126585, 126588 }, + { 126590, 126590 }, + { 126592, 126601 }, + { 126603, 126619 }, + { 126625, 126627 }, + { 126629, 126633 }, + { 126635, 126651 }, + { 126704, 126705 }, +}; +static const URange16 Armenian_range16[] = { + { 1329, 1366 }, + { 1369, 1416 }, + { 1418, 1418 }, + { 1421, 1423 }, + { 64275, 64279 }, +}; +static const URange32 Avestan_range32[] = { + { 68352, 68405 }, + { 68409, 68415 }, +}; +static const URange16 Balinese_range16[] = { + { 6912, 6987 }, + { 6992, 7036 }, +}; +static const URange16 Bamum_range16[] = { + { 42656, 42743 }, +}; +static const URange32 Bamum_range32[] = { + { 92160, 92728 }, +}; +static const URange32 Bassa_Vah_range32[] = { + { 92880, 92909 }, + { 92912, 92917 }, +}; +static const URange16 Batak_range16[] = { + { 7104, 7155 }, + { 7164, 7167 }, +}; +static const URange16 Bengali_range16[] = { + { 2432, 2435 }, + { 2437, 2444 }, + { 2447, 2448 }, + { 2451, 2472 }, + { 2474, 2480 }, + { 2482, 2482 }, + { 2486, 2489 }, + { 2492, 2500 }, + { 2503, 2504 }, + { 2507, 2510 }, + { 2519, 2519 }, + { 2524, 2525 }, + { 2527, 2531 }, + { 2534, 2558 }, +}; +static const URange32 Bhaiksuki_range32[] = { + { 72704, 72712 }, + { 72714, 72758 }, + { 72760, 72773 }, + { 72784, 72812 }, +}; +static const URange16 Bopomofo_range16[] = { + { 746, 747 }, + { 12549, 12591 }, + { 12704, 12730 }, +}; +static const URange32 Brahmi_range32[] = { + { 69632, 69709 }, + { 69714, 69743 }, + { 69759, 69759 }, +}; +static const URange16 Braille_range16[] = { + { 10240, 10495 }, +}; +static const URange16 Buginese_range16[] = { + { 6656, 6683 }, + { 6686, 6687 }, +}; +static const URange16 Buhid_range16[] = { + { 5952, 5971 }, +}; +static const URange16 Canadian_Aboriginal_range16[] = { + { 5120, 5759 }, + { 6320, 6389 }, +}; +static const URange32 Carian_range32[] = { + { 66208, 66256 }, +}; +static const URange32 Caucasian_Albanian_range32[] = { + { 66864, 66915 }, + { 66927, 66927 }, +}; +static const URange32 Chakma_range32[] = { + { 69888, 69940 }, + { 69942, 69958 }, +}; +static const URange16 Cham_range16[] = { + { 43520, 43574 }, + { 43584, 43597 }, + { 43600, 43609 }, + { 43612, 43615 }, +}; +static const URange16 Cherokee_range16[] = { + { 5024, 5109 }, + { 5112, 5117 }, + { 43888, 43967 }, +}; +static const URange16 Common_range16[] = { + { 0, 64 }, + { 91, 96 }, + { 123, 169 }, + { 171, 185 }, + { 187, 191 }, + { 215, 215 }, + { 247, 247 }, + { 697, 735 }, + { 741, 745 }, + { 748, 767 }, + { 884, 884 }, + { 894, 894 }, + { 901, 901 }, + { 903, 903 }, + { 1417, 1417 }, + { 1541, 1541 }, + { 1548, 1548 }, + { 1563, 1563 }, + { 1567, 1567 }, + { 1600, 1600 }, + { 1757, 1757 }, + { 2274, 2274 }, + { 2404, 2405 }, + { 3647, 3647 }, + { 4053, 4056 }, + { 4347, 4347 }, + { 5867, 5869 }, + { 5941, 5942 }, + { 6146, 6147 }, + { 6149, 6149 }, + { 7379, 7379 }, + { 7393, 7393 }, + { 7401, 7404 }, + { 7406, 7411 }, + { 7413, 7415 }, + { 7418, 7418 }, + { 8192, 8203 }, + { 8206, 8292 }, + { 8294, 8304 }, + { 8308, 8318 }, + { 8320, 8334 }, + { 8352, 8383 }, + { 8448, 8485 }, + { 8487, 8489 }, + { 8492, 8497 }, + { 8499, 8525 }, + { 8527, 8543 }, + { 8585, 8587 }, + { 8592, 9254 }, + { 9280, 9290 }, + { 9312, 10239 }, + { 10496, 11123 }, + { 11126, 11157 }, + { 11160, 11263 }, + { 11776, 11855 }, + { 12272, 12283 }, + { 12288, 12292 }, + { 12294, 12294 }, + { 12296, 12320 }, + { 12336, 12343 }, + { 12348, 12351 }, + { 12443, 12444 }, + { 12448, 12448 }, + { 12539, 12540 }, + { 12688, 12703 }, + { 12736, 12771 }, + { 12832, 12895 }, + { 12927, 13007 }, + { 13055, 13055 }, + { 13144, 13311 }, + { 19904, 19967 }, + { 42752, 42785 }, + { 42888, 42890 }, + { 43056, 43065 }, + { 43310, 43310 }, + { 43471, 43471 }, + { 43867, 43867 }, + { 64830, 64831 }, + { 65040, 65049 }, + { 65072, 65106 }, + { 65108, 65126 }, + { 65128, 65131 }, + { 65279, 65279 }, + { 65281, 65312 }, + { 65339, 65344 }, + { 65371, 65381 }, + { 65392, 65392 }, + { 65438, 65439 }, + { 65504, 65510 }, + { 65512, 65518 }, + { 65529, 65533 }, +}; +static const URange32 Common_range32[] = { + { 65792, 65794 }, + { 65799, 65843 }, + { 65847, 65855 }, + { 65936, 65947 }, + { 66000, 66044 }, + { 66273, 66299 }, + { 94178, 94179 }, + { 113824, 113827 }, + { 118784, 119029 }, + { 119040, 119078 }, + { 119081, 119142 }, + { 119146, 119162 }, + { 119171, 119172 }, + { 119180, 119209 }, + { 119214, 119272 }, + { 119520, 119539 }, + { 119552, 119638 }, + { 119648, 119672 }, + { 119808, 119892 }, + { 119894, 119964 }, + { 119966, 119967 }, + { 119970, 119970 }, + { 119973, 119974 }, + { 119977, 119980 }, + { 119982, 119993 }, + { 119995, 119995 }, + { 119997, 120003 }, + { 120005, 120069 }, + { 120071, 120074 }, + { 120077, 120084 }, + { 120086, 120092 }, + { 120094, 120121 }, + { 120123, 120126 }, + { 120128, 120132 }, + { 120134, 120134 }, + { 120138, 120144 }, + { 120146, 120485 }, + { 120488, 120779 }, + { 120782, 120831 }, + { 126065, 126132 }, + { 126209, 126269 }, + { 126976, 127019 }, + { 127024, 127123 }, + { 127136, 127150 }, + { 127153, 127167 }, + { 127169, 127183 }, + { 127185, 127221 }, + { 127232, 127244 }, + { 127248, 127340 }, + { 127344, 127404 }, + { 127462, 127487 }, + { 127489, 127490 }, + { 127504, 127547 }, + { 127552, 127560 }, + { 127568, 127569 }, + { 127584, 127589 }, + { 127744, 128725 }, + { 128736, 128748 }, + { 128752, 128762 }, + { 128768, 128883 }, + { 128896, 128984 }, + { 128992, 129003 }, + { 129024, 129035 }, + { 129040, 129095 }, + { 129104, 129113 }, + { 129120, 129159 }, + { 129168, 129197 }, + { 129280, 129291 }, + { 129293, 129393 }, + { 129395, 129398 }, + { 129402, 129442 }, + { 129445, 129450 }, + { 129454, 129482 }, + { 129485, 129619 }, + { 129632, 129645 }, + { 129648, 129651 }, + { 129656, 129658 }, + { 129664, 129666 }, + { 129680, 129685 }, + { 917505, 917505 }, + { 917536, 917631 }, +}; +static const URange16 Coptic_range16[] = { + { 994, 1007 }, + { 11392, 11507 }, + { 11513, 11519 }, +}; +static const URange32 Cuneiform_range32[] = { + { 73728, 74649 }, + { 74752, 74862 }, + { 74864, 74868 }, + { 74880, 75075 }, +}; +static const URange32 Cypriot_range32[] = { + { 67584, 67589 }, + { 67592, 67592 }, + { 67594, 67637 }, + { 67639, 67640 }, + { 67644, 67644 }, + { 67647, 67647 }, +}; +static const URange16 Cyrillic_range16[] = { + { 1024, 1156 }, + { 1159, 1327 }, + { 7296, 7304 }, + { 7467, 7467 }, + { 7544, 7544 }, + { 11744, 11775 }, + { 42560, 42655 }, + { 65070, 65071 }, +}; +static const URange32 Deseret_range32[] = { + { 66560, 66639 }, +}; +static const URange16 Devanagari_range16[] = { + { 2304, 2384 }, + { 2389, 2403 }, + { 2406, 2431 }, + { 43232, 43263 }, +}; +static const URange32 Dogra_range32[] = { + { 71680, 71739 }, +}; +static const URange32 Duployan_range32[] = { + { 113664, 113770 }, + { 113776, 113788 }, + { 113792, 113800 }, + { 113808, 113817 }, + { 113820, 113823 }, +}; +static const URange32 Egyptian_Hieroglyphs_range32[] = { + { 77824, 78894 }, + { 78896, 78904 }, +}; +static const URange32 Elbasan_range32[] = { + { 66816, 66855 }, +}; +static const URange32 Elymaic_range32[] = { + { 69600, 69622 }, +}; +static const URange16 Ethiopic_range16[] = { + { 4608, 4680 }, + { 4682, 4685 }, + { 4688, 4694 }, + { 4696, 4696 }, + { 4698, 4701 }, + { 4704, 4744 }, + { 4746, 4749 }, + { 4752, 4784 }, + { 4786, 4789 }, + { 4792, 4798 }, + { 4800, 4800 }, + { 4802, 4805 }, + { 4808, 4822 }, + { 4824, 4880 }, + { 4882, 4885 }, + { 4888, 4954 }, + { 4957, 4988 }, + { 4992, 5017 }, + { 11648, 11670 }, + { 11680, 11686 }, + { 11688, 11694 }, + { 11696, 11702 }, + { 11704, 11710 }, + { 11712, 11718 }, + { 11720, 11726 }, + { 11728, 11734 }, + { 11736, 11742 }, + { 43777, 43782 }, + { 43785, 43790 }, + { 43793, 43798 }, + { 43808, 43814 }, + { 43816, 43822 }, +}; +static const URange16 Georgian_range16[] = { + { 4256, 4293 }, + { 4295, 4295 }, + { 4301, 4301 }, + { 4304, 4346 }, + { 4348, 4351 }, + { 7312, 7354 }, + { 7357, 7359 }, + { 11520, 11557 }, + { 11559, 11559 }, + { 11565, 11565 }, +}; +static const URange16 Glagolitic_range16[] = { + { 11264, 11310 }, + { 11312, 11358 }, +}; +static const URange32 Glagolitic_range32[] = { + { 122880, 122886 }, + { 122888, 122904 }, + { 122907, 122913 }, + { 122915, 122916 }, + { 122918, 122922 }, +}; +static const URange32 Gothic_range32[] = { + { 66352, 66378 }, +}; +static const URange32 Grantha_range32[] = { + { 70400, 70403 }, + { 70405, 70412 }, + { 70415, 70416 }, + { 70419, 70440 }, + { 70442, 70448 }, + { 70450, 70451 }, + { 70453, 70457 }, + { 70460, 70468 }, + { 70471, 70472 }, + { 70475, 70477 }, + { 70480, 70480 }, + { 70487, 70487 }, + { 70493, 70499 }, + { 70502, 70508 }, + { 70512, 70516 }, +}; +static const URange16 Greek_range16[] = { + { 880, 883 }, + { 885, 887 }, + { 890, 893 }, + { 895, 895 }, + { 900, 900 }, + { 902, 902 }, + { 904, 906 }, + { 908, 908 }, + { 910, 929 }, + { 931, 993 }, + { 1008, 1023 }, + { 7462, 7466 }, + { 7517, 7521 }, + { 7526, 7530 }, + { 7615, 7615 }, + { 7936, 7957 }, + { 7960, 7965 }, + { 7968, 8005 }, + { 8008, 8013 }, + { 8016, 8023 }, + { 8025, 8025 }, + { 8027, 8027 }, + { 8029, 8029 }, + { 8031, 8061 }, + { 8064, 8116 }, + { 8118, 8132 }, + { 8134, 8147 }, + { 8150, 8155 }, + { 8157, 8175 }, + { 8178, 8180 }, + { 8182, 8190 }, + { 8486, 8486 }, + { 43877, 43877 }, +}; +static const URange32 Greek_range32[] = { + { 65856, 65934 }, + { 65952, 65952 }, + { 119296, 119365 }, +}; +static const URange16 Gujarati_range16[] = { + { 2689, 2691 }, + { 2693, 2701 }, + { 2703, 2705 }, + { 2707, 2728 }, + { 2730, 2736 }, + { 2738, 2739 }, + { 2741, 2745 }, + { 2748, 2757 }, + { 2759, 2761 }, + { 2763, 2765 }, + { 2768, 2768 }, + { 2784, 2787 }, + { 2790, 2801 }, + { 2809, 2815 }, +}; +static const URange32 Gunjala_Gondi_range32[] = { + { 73056, 73061 }, + { 73063, 73064 }, + { 73066, 73102 }, + { 73104, 73105 }, + { 73107, 73112 }, + { 73120, 73129 }, +}; +static const URange16 Gurmukhi_range16[] = { + { 2561, 2563 }, + { 2565, 2570 }, + { 2575, 2576 }, + { 2579, 2600 }, + { 2602, 2608 }, + { 2610, 2611 }, + { 2613, 2614 }, + { 2616, 2617 }, + { 2620, 2620 }, + { 2622, 2626 }, + { 2631, 2632 }, + { 2635, 2637 }, + { 2641, 2641 }, + { 2649, 2652 }, + { 2654, 2654 }, + { 2662, 2678 }, +}; +static const URange16 Han_range16[] = { + { 11904, 11929 }, + { 11931, 12019 }, + { 12032, 12245 }, + { 12293, 12293 }, + { 12295, 12295 }, + { 12321, 12329 }, + { 12344, 12347 }, + { 13312, 19893 }, + { 19968, 40943 }, + { 63744, 64109 }, + { 64112, 64217 }, +}; +static const URange32 Han_range32[] = { + { 131072, 173782 }, + { 173824, 177972 }, + { 177984, 178205 }, + { 178208, 183969 }, + { 183984, 191456 }, + { 194560, 195101 }, +}; +static const URange16 Hangul_range16[] = { + { 4352, 4607 }, + { 12334, 12335 }, + { 12593, 12686 }, + { 12800, 12830 }, + { 12896, 12926 }, + { 43360, 43388 }, + { 44032, 55203 }, + { 55216, 55238 }, + { 55243, 55291 }, + { 65440, 65470 }, + { 65474, 65479 }, + { 65482, 65487 }, + { 65490, 65495 }, + { 65498, 65500 }, +}; +static const URange32 Hanifi_Rohingya_range32[] = { + { 68864, 68903 }, + { 68912, 68921 }, +}; +static const URange16 Hanunoo_range16[] = { + { 5920, 5940 }, +}; +static const URange32 Hatran_range32[] = { + { 67808, 67826 }, + { 67828, 67829 }, + { 67835, 67839 }, +}; +static const URange16 Hebrew_range16[] = { + { 1425, 1479 }, + { 1488, 1514 }, + { 1519, 1524 }, + { 64285, 64310 }, + { 64312, 64316 }, + { 64318, 64318 }, + { 64320, 64321 }, + { 64323, 64324 }, + { 64326, 64335 }, +}; +static const URange16 Hiragana_range16[] = { + { 12353, 12438 }, + { 12445, 12447 }, +}; +static const URange32 Hiragana_range32[] = { + { 110593, 110878 }, + { 110928, 110930 }, + { 127488, 127488 }, +}; +static const URange32 Imperial_Aramaic_range32[] = { + { 67648, 67669 }, + { 67671, 67679 }, +}; +static const URange16 Inherited_range16[] = { + { 768, 879 }, + { 1157, 1158 }, + { 1611, 1621 }, + { 1648, 1648 }, + { 2385, 2388 }, + { 6832, 6846 }, + { 7376, 7378 }, + { 7380, 7392 }, + { 7394, 7400 }, + { 7405, 7405 }, + { 7412, 7412 }, + { 7416, 7417 }, + { 7616, 7673 }, + { 7675, 7679 }, + { 8204, 8205 }, + { 8400, 8432 }, + { 12330, 12333 }, + { 12441, 12442 }, + { 65024, 65039 }, + { 65056, 65069 }, +}; +static const URange32 Inherited_range32[] = { + { 66045, 66045 }, + { 66272, 66272 }, + { 70459, 70459 }, + { 119143, 119145 }, + { 119163, 119170 }, + { 119173, 119179 }, + { 119210, 119213 }, + { 917760, 917999 }, +}; +static const URange32 Inscriptional_Pahlavi_range32[] = { + { 68448, 68466 }, + { 68472, 68479 }, +}; +static const URange32 Inscriptional_Parthian_range32[] = { + { 68416, 68437 }, + { 68440, 68447 }, +}; +static const URange16 Javanese_range16[] = { + { 43392, 43469 }, + { 43472, 43481 }, + { 43486, 43487 }, +}; +static const URange32 Kaithi_range32[] = { + { 69760, 69825 }, + { 69837, 69837 }, +}; +static const URange16 Kannada_range16[] = { + { 3200, 3212 }, + { 3214, 3216 }, + { 3218, 3240 }, + { 3242, 3251 }, + { 3253, 3257 }, + { 3260, 3268 }, + { 3270, 3272 }, + { 3274, 3277 }, + { 3285, 3286 }, + { 3294, 3294 }, + { 3296, 3299 }, + { 3302, 3311 }, + { 3313, 3314 }, +}; +static const URange16 Katakana_range16[] = { + { 12449, 12538 }, + { 12541, 12543 }, + { 12784, 12799 }, + { 13008, 13054 }, + { 13056, 13143 }, + { 65382, 65391 }, + { 65393, 65437 }, +}; +static const URange32 Katakana_range32[] = { + { 110592, 110592 }, + { 110948, 110951 }, +}; +static const URange16 Kayah_Li_range16[] = { + { 43264, 43309 }, + { 43311, 43311 }, +}; +static const URange32 Kharoshthi_range32[] = { + { 68096, 68099 }, + { 68101, 68102 }, + { 68108, 68115 }, + { 68117, 68119 }, + { 68121, 68149 }, + { 68152, 68154 }, + { 68159, 68168 }, + { 68176, 68184 }, +}; +static const URange16 Khmer_range16[] = { + { 6016, 6109 }, + { 6112, 6121 }, + { 6128, 6137 }, + { 6624, 6655 }, +}; +static const URange32 Khojki_range32[] = { + { 70144, 70161 }, + { 70163, 70206 }, +}; +static const URange32 Khudawadi_range32[] = { + { 70320, 70378 }, + { 70384, 70393 }, +}; +static const URange16 Lao_range16[] = { + { 3713, 3714 }, + { 3716, 3716 }, + { 3718, 3722 }, + { 3724, 3747 }, + { 3749, 3749 }, + { 3751, 3773 }, + { 3776, 3780 }, + { 3782, 3782 }, + { 3784, 3789 }, + { 3792, 3801 }, + { 3804, 3807 }, +}; +static const URange16 Latin_range16[] = { + { 65, 90 }, + { 97, 122 }, + { 170, 170 }, + { 186, 186 }, + { 192, 214 }, + { 216, 246 }, + { 248, 696 }, + { 736, 740 }, + { 7424, 7461 }, + { 7468, 7516 }, + { 7522, 7525 }, + { 7531, 7543 }, + { 7545, 7614 }, + { 7680, 7935 }, + { 8305, 8305 }, + { 8319, 8319 }, + { 8336, 8348 }, + { 8490, 8491 }, + { 8498, 8498 }, + { 8526, 8526 }, + { 8544, 8584 }, + { 11360, 11391 }, + { 42786, 42887 }, + { 42891, 42943 }, + { 42946, 42950 }, + { 42999, 43007 }, + { 43824, 43866 }, + { 43868, 43876 }, + { 43878, 43879 }, + { 64256, 64262 }, + { 65313, 65338 }, + { 65345, 65370 }, +}; +static const URange16 Lepcha_range16[] = { + { 7168, 7223 }, + { 7227, 7241 }, + { 7245, 7247 }, +}; +static const URange16 Limbu_range16[] = { + { 6400, 6430 }, + { 6432, 6443 }, + { 6448, 6459 }, + { 6464, 6464 }, + { 6468, 6479 }, +}; +static const URange32 Linear_A_range32[] = { + { 67072, 67382 }, + { 67392, 67413 }, + { 67424, 67431 }, +}; +static const URange32 Linear_B_range32[] = { + { 65536, 65547 }, + { 65549, 65574 }, + { 65576, 65594 }, + { 65596, 65597 }, + { 65599, 65613 }, + { 65616, 65629 }, + { 65664, 65786 }, +}; +static const URange16 Lisu_range16[] = { + { 42192, 42239 }, +}; +static const URange32 Lycian_range32[] = { + { 66176, 66204 }, +}; +static const URange32 Lydian_range32[] = { + { 67872, 67897 }, + { 67903, 67903 }, +}; +static const URange32 Mahajani_range32[] = { + { 69968, 70006 }, +}; +static const URange32 Makasar_range32[] = { + { 73440, 73464 }, +}; +static const URange16 Malayalam_range16[] = { + { 3328, 3331 }, + { 3333, 3340 }, + { 3342, 3344 }, + { 3346, 3396 }, + { 3398, 3400 }, + { 3402, 3407 }, + { 3412, 3427 }, + { 3430, 3455 }, +}; +static const URange16 Mandaic_range16[] = { + { 2112, 2139 }, + { 2142, 2142 }, +}; +static const URange32 Manichaean_range32[] = { + { 68288, 68326 }, + { 68331, 68342 }, +}; +static const URange32 Marchen_range32[] = { + { 72816, 72847 }, + { 72850, 72871 }, + { 72873, 72886 }, +}; +static const URange32 Masaram_Gondi_range32[] = { + { 72960, 72966 }, + { 72968, 72969 }, + { 72971, 73014 }, + { 73018, 73018 }, + { 73020, 73021 }, + { 73023, 73031 }, + { 73040, 73049 }, +}; +static const URange32 Medefaidrin_range32[] = { + { 93760, 93850 }, +}; +static const URange16 Meetei_Mayek_range16[] = { + { 43744, 43766 }, + { 43968, 44013 }, + { 44016, 44025 }, +}; +static const URange32 Mende_Kikakui_range32[] = { + { 124928, 125124 }, + { 125127, 125142 }, +}; +static const URange32 Meroitic_Cursive_range32[] = { + { 68000, 68023 }, + { 68028, 68047 }, + { 68050, 68095 }, +}; +static const URange32 Meroitic_Hieroglyphs_range32[] = { + { 67968, 67999 }, +}; +static const URange32 Miao_range32[] = { + { 93952, 94026 }, + { 94031, 94087 }, + { 94095, 94111 }, +}; +static const URange32 Modi_range32[] = { + { 71168, 71236 }, + { 71248, 71257 }, +}; +static const URange16 Mongolian_range16[] = { + { 6144, 6145 }, + { 6148, 6148 }, + { 6150, 6158 }, + { 6160, 6169 }, + { 6176, 6264 }, + { 6272, 6314 }, +}; +static const URange32 Mongolian_range32[] = { + { 71264, 71276 }, +}; +static const URange32 Mro_range32[] = { + { 92736, 92766 }, + { 92768, 92777 }, + { 92782, 92783 }, +}; +static const URange32 Multani_range32[] = { + { 70272, 70278 }, + { 70280, 70280 }, + { 70282, 70285 }, + { 70287, 70301 }, + { 70303, 70313 }, +}; +static const URange16 Myanmar_range16[] = { + { 4096, 4255 }, + { 43488, 43518 }, + { 43616, 43647 }, +}; +static const URange32 Nabataean_range32[] = { + { 67712, 67742 }, + { 67751, 67759 }, +}; +static const URange32 Nandinagari_range32[] = { + { 72096, 72103 }, + { 72106, 72151 }, + { 72154, 72164 }, +}; +static const URange16 New_Tai_Lue_range16[] = { + { 6528, 6571 }, + { 6576, 6601 }, + { 6608, 6618 }, + { 6622, 6623 }, +}; +static const URange32 Newa_range32[] = { + { 70656, 70745 }, + { 70747, 70747 }, + { 70749, 70751 }, +}; +static const URange16 Nko_range16[] = { + { 1984, 2042 }, + { 2045, 2047 }, +}; +static const URange32 Nushu_range32[] = { + { 94177, 94177 }, + { 110960, 111355 }, +}; +static const URange32 Nyiakeng_Puachue_Hmong_range32[] = { + { 123136, 123180 }, + { 123184, 123197 }, + { 123200, 123209 }, + { 123214, 123215 }, +}; +static const URange16 Ogham_range16[] = { + { 5760, 5788 }, +}; +static const URange16 Ol_Chiki_range16[] = { + { 7248, 7295 }, +}; +static const URange32 Old_Hungarian_range32[] = { + { 68736, 68786 }, + { 68800, 68850 }, + { 68858, 68863 }, +}; +static const URange32 Old_Italic_range32[] = { + { 66304, 66339 }, + { 66349, 66351 }, +}; +static const URange32 Old_North_Arabian_range32[] = { + { 68224, 68255 }, +}; +static const URange32 Old_Permic_range32[] = { + { 66384, 66426 }, +}; +static const URange32 Old_Persian_range32[] = { + { 66464, 66499 }, + { 66504, 66517 }, +}; +static const URange32 Old_Sogdian_range32[] = { + { 69376, 69415 }, +}; +static const URange32 Old_South_Arabian_range32[] = { + { 68192, 68223 }, +}; +static const URange32 Old_Turkic_range32[] = { + { 68608, 68680 }, +}; +static const URange16 Oriya_range16[] = { + { 2817, 2819 }, + { 2821, 2828 }, + { 2831, 2832 }, + { 2835, 2856 }, + { 2858, 2864 }, + { 2866, 2867 }, + { 2869, 2873 }, + { 2876, 2884 }, + { 2887, 2888 }, + { 2891, 2893 }, + { 2902, 2903 }, + { 2908, 2909 }, + { 2911, 2915 }, + { 2918, 2935 }, +}; +static const URange32 Osage_range32[] = { + { 66736, 66771 }, + { 66776, 66811 }, +}; +static const URange32 Osmanya_range32[] = { + { 66688, 66717 }, + { 66720, 66729 }, +}; +static const URange32 Pahawh_Hmong_range32[] = { + { 92928, 92997 }, + { 93008, 93017 }, + { 93019, 93025 }, + { 93027, 93047 }, + { 93053, 93071 }, +}; +static const URange32 Palmyrene_range32[] = { + { 67680, 67711 }, +}; +static const URange32 Pau_Cin_Hau_range32[] = { + { 72384, 72440 }, +}; +static const URange16 Phags_Pa_range16[] = { + { 43072, 43127 }, +}; +static const URange32 Phoenician_range32[] = { + { 67840, 67867 }, + { 67871, 67871 }, +}; +static const URange32 Psalter_Pahlavi_range32[] = { + { 68480, 68497 }, + { 68505, 68508 }, + { 68521, 68527 }, +}; +static const URange16 Rejang_range16[] = { + { 43312, 43347 }, + { 43359, 43359 }, +}; +static const URange16 Runic_range16[] = { + { 5792, 5866 }, + { 5870, 5880 }, +}; +static const URange16 Samaritan_range16[] = { + { 2048, 2093 }, + { 2096, 2110 }, +}; +static const URange16 Saurashtra_range16[] = { + { 43136, 43205 }, + { 43214, 43225 }, +}; +static const URange32 Sharada_range32[] = { + { 70016, 70093 }, + { 70096, 70111 }, +}; +static const URange32 Shavian_range32[] = { + { 66640, 66687 }, +}; +static const URange32 Siddham_range32[] = { + { 71040, 71093 }, + { 71096, 71133 }, +}; +static const URange32 SignWriting_range32[] = { + { 120832, 121483 }, + { 121499, 121503 }, + { 121505, 121519 }, +}; +static const URange16 Sinhala_range16[] = { + { 3458, 3459 }, + { 3461, 3478 }, + { 3482, 3505 }, + { 3507, 3515 }, + { 3517, 3517 }, + { 3520, 3526 }, + { 3530, 3530 }, + { 3535, 3540 }, + { 3542, 3542 }, + { 3544, 3551 }, + { 3558, 3567 }, + { 3570, 3572 }, +}; +static const URange32 Sinhala_range32[] = { + { 70113, 70132 }, +}; +static const URange32 Sogdian_range32[] = { + { 69424, 69465 }, +}; +static const URange32 Sora_Sompeng_range32[] = { + { 69840, 69864 }, + { 69872, 69881 }, +}; +static const URange32 Soyombo_range32[] = { + { 72272, 72354 }, +}; +static const URange16 Sundanese_range16[] = { + { 7040, 7103 }, + { 7360, 7367 }, +}; +static const URange16 Syloti_Nagri_range16[] = { + { 43008, 43051 }, +}; +static const URange16 Syriac_range16[] = { + { 1792, 1805 }, + { 1807, 1866 }, + { 1869, 1871 }, + { 2144, 2154 }, +}; +static const URange16 Tagalog_range16[] = { + { 5888, 5900 }, + { 5902, 5908 }, +}; +static const URange16 Tagbanwa_range16[] = { + { 5984, 5996 }, + { 5998, 6000 }, + { 6002, 6003 }, +}; +static const URange16 Tai_Le_range16[] = { + { 6480, 6509 }, + { 6512, 6516 }, +}; +static const URange16 Tai_Tham_range16[] = { + { 6688, 6750 }, + { 6752, 6780 }, + { 6783, 6793 }, + { 6800, 6809 }, + { 6816, 6829 }, +}; +static const URange16 Tai_Viet_range16[] = { + { 43648, 43714 }, + { 43739, 43743 }, +}; +static const URange32 Takri_range32[] = { + { 71296, 71352 }, + { 71360, 71369 }, +}; +static const URange16 Tamil_range16[] = { + { 2946, 2947 }, + { 2949, 2954 }, + { 2958, 2960 }, + { 2962, 2965 }, + { 2969, 2970 }, + { 2972, 2972 }, + { 2974, 2975 }, + { 2979, 2980 }, + { 2984, 2986 }, + { 2990, 3001 }, + { 3006, 3010 }, + { 3014, 3016 }, + { 3018, 3021 }, + { 3024, 3024 }, + { 3031, 3031 }, + { 3046, 3066 }, +}; +static const URange32 Tamil_range32[] = { + { 73664, 73713 }, + { 73727, 73727 }, +}; +static const URange32 Tangut_range32[] = { + { 94176, 94176 }, + { 94208, 100343 }, + { 100352, 101106 }, +}; +static const URange16 Telugu_range16[] = { + { 3072, 3084 }, + { 3086, 3088 }, + { 3090, 3112 }, + { 3114, 3129 }, + { 3133, 3140 }, + { 3142, 3144 }, + { 3146, 3149 }, + { 3157, 3158 }, + { 3160, 3162 }, + { 3168, 3171 }, + { 3174, 3183 }, + { 3191, 3199 }, +}; +static const URange16 Thaana_range16[] = { + { 1920, 1969 }, +}; +static const URange16 Thai_range16[] = { + { 3585, 3642 }, + { 3648, 3675 }, +}; +static const URange16 Tibetan_range16[] = { + { 3840, 3911 }, + { 3913, 3948 }, + { 3953, 3991 }, + { 3993, 4028 }, + { 4030, 4044 }, + { 4046, 4052 }, + { 4057, 4058 }, +}; +static const URange16 Tifinagh_range16[] = { + { 11568, 11623 }, + { 11631, 11632 }, + { 11647, 11647 }, +}; +static const URange32 Tirhuta_range32[] = { + { 70784, 70855 }, + { 70864, 70873 }, +}; +static const URange32 Ugaritic_range32[] = { + { 66432, 66461 }, + { 66463, 66463 }, +}; +static const URange16 Vai_range16[] = { + { 42240, 42539 }, +}; +static const URange32 Wancho_range32[] = { + { 123584, 123641 }, + { 123647, 123647 }, +}; +static const URange32 Warang_Citi_range32[] = { + { 71840, 71922 }, + { 71935, 71935 }, +}; +static const URange16 Yi_range16[] = { + { 40960, 42124 }, + { 42128, 42182 }, +}; +static const URange32 Zanabazar_Square_range32[] = { + { 72192, 72263 }, +}; +// 3987 16-bit ranges, 1525 32-bit ranges +const UGroup unicode_groups[] = { + { "Adlam", +1, 0, 0, Adlam_range32, 3 }, + { "Ahom", +1, 0, 0, Ahom_range32, 3 }, + { "Anatolian_Hieroglyphs", +1, 0, 0, Anatolian_Hieroglyphs_range32, 1 }, + { "Arabic", +1, Arabic_range16, 22, Arabic_range32, 35 }, + { "Armenian", +1, Armenian_range16, 5, 0, 0 }, + { "Avestan", +1, 0, 0, Avestan_range32, 2 }, + { "Balinese", +1, Balinese_range16, 2, 0, 0 }, + { "Bamum", +1, Bamum_range16, 1, Bamum_range32, 1 }, + { "Bassa_Vah", +1, 0, 0, Bassa_Vah_range32, 2 }, + { "Batak", +1, Batak_range16, 2, 0, 0 }, + { "Bengali", +1, Bengali_range16, 14, 0, 0 }, + { "Bhaiksuki", +1, 0, 0, Bhaiksuki_range32, 4 }, + { "Bopomofo", +1, Bopomofo_range16, 3, 0, 0 }, + { "Brahmi", +1, 0, 0, Brahmi_range32, 3 }, + { "Braille", +1, Braille_range16, 1, 0, 0 }, + { "Buginese", +1, Buginese_range16, 2, 0, 0 }, + { "Buhid", +1, Buhid_range16, 1, 0, 0 }, + { "C", +1, C_range16, 16, C_range32, 9 }, + { "Canadian_Aboriginal", +1, Canadian_Aboriginal_range16, 2, 0, 0 }, + { "Carian", +1, 0, 0, Carian_range32, 1 }, + { "Caucasian_Albanian", +1, 0, 0, Caucasian_Albanian_range32, 2 }, + { "Cc", +1, Cc_range16, 2, 0, 0 }, + { "Cf", +1, Cf_range16, 13, Cf_range32, 7 }, + { "Chakma", +1, 0, 0, Chakma_range32, 2 }, + { "Cham", +1, Cham_range16, 4, 0, 0 }, + { "Cherokee", +1, Cherokee_range16, 3, 0, 0 }, + { "Co", +1, Co_range16, 1, Co_range32, 2 }, + { "Common", +1, Common_range16, 91, Common_range32, 81 }, + { "Coptic", +1, Coptic_range16, 3, 0, 0 }, + { "Cs", +1, Cs_range16, 1, 0, 0 }, + { "Cuneiform", +1, 0, 0, Cuneiform_range32, 4 }, + { "Cypriot", +1, 0, 0, Cypriot_range32, 6 }, + { "Cyrillic", +1, Cyrillic_range16, 8, 0, 0 }, + { "Deseret", +1, 0, 0, Deseret_range32, 1 }, + { "Devanagari", +1, Devanagari_range16, 4, 0, 0 }, + { "Dogra", +1, 0, 0, Dogra_range32, 1 }, + { "Duployan", +1, 0, 0, Duployan_range32, 5 }, + { "Egyptian_Hieroglyphs", +1, 0, 0, Egyptian_Hieroglyphs_range32, 2 }, + { "Elbasan", +1, 0, 0, Elbasan_range32, 1 }, + { "Elymaic", +1, 0, 0, Elymaic_range32, 1 }, + { "Ethiopic", +1, Ethiopic_range16, 32, 0, 0 }, + { "Georgian", +1, Georgian_range16, 10, 0, 0 }, + { "Glagolitic", +1, Glagolitic_range16, 2, Glagolitic_range32, 5 }, + { "Gothic", +1, 0, 0, Gothic_range32, 1 }, + { "Grantha", +1, 0, 0, Grantha_range32, 15 }, + { "Greek", +1, Greek_range16, 33, Greek_range32, 3 }, + { "Gujarati", +1, Gujarati_range16, 14, 0, 0 }, + { "Gunjala_Gondi", +1, 0, 0, Gunjala_Gondi_range32, 6 }, + { "Gurmukhi", +1, Gurmukhi_range16, 16, 0, 0 }, + { "Han", +1, Han_range16, 11, Han_range32, 6 }, + { "Hangul", +1, Hangul_range16, 14, 0, 0 }, + { "Hanifi_Rohingya", +1, 0, 0, Hanifi_Rohingya_range32, 2 }, + { "Hanunoo", +1, Hanunoo_range16, 1, 0, 0 }, + { "Hatran", +1, 0, 0, Hatran_range32, 3 }, + { "Hebrew", +1, Hebrew_range16, 9, 0, 0 }, + { "Hiragana", +1, Hiragana_range16, 2, Hiragana_range32, 3 }, + { "Imperial_Aramaic", +1, 0, 0, Imperial_Aramaic_range32, 2 }, + { "Inherited", +1, Inherited_range16, 20, Inherited_range32, 8 }, + { "Inscriptional_Pahlavi", +1, 0, 0, Inscriptional_Pahlavi_range32, 2 }, + { "Inscriptional_Parthian", +1, 0, 0, Inscriptional_Parthian_range32, 2 }, + { "Javanese", +1, Javanese_range16, 3, 0, 0 }, + { "Kaithi", +1, 0, 0, Kaithi_range32, 2 }, + { "Kannada", +1, Kannada_range16, 13, 0, 0 }, + { "Katakana", +1, Katakana_range16, 7, Katakana_range32, 2 }, + { "Kayah_Li", +1, Kayah_Li_range16, 2, 0, 0 }, + { "Kharoshthi", +1, 0, 0, Kharoshthi_range32, 8 }, + { "Khmer", +1, Khmer_range16, 4, 0, 0 }, + { "Khojki", +1, 0, 0, Khojki_range32, 2 }, + { "Khudawadi", +1, 0, 0, Khudawadi_range32, 2 }, + { "L", +1, L_range16, 380, L_range32, 229 }, + { "Lao", +1, Lao_range16, 11, 0, 0 }, + { "Latin", +1, Latin_range16, 32, 0, 0 }, + { "Lepcha", +1, Lepcha_range16, 3, 0, 0 }, + { "Limbu", +1, Limbu_range16, 5, 0, 0 }, + { "Linear_A", +1, 0, 0, Linear_A_range32, 3 }, + { "Linear_B", +1, 0, 0, Linear_B_range32, 7 }, + { "Lisu", +1, Lisu_range16, 1, 0, 0 }, + { "Ll", +1, Ll_range16, 608, Ll_range32, 34 }, + { "Lm", +1, Lm_range16, 54, Lm_range32, 6 }, + { "Lo", +1, Lo_range16, 290, Lo_range32, 186 }, + { "Lt", +1, Lt_range16, 10, 0, 0 }, + { "Lu", +1, Lu_range16, 599, Lu_range32, 37 }, + { "Lycian", +1, 0, 0, Lycian_range32, 1 }, + { "Lydian", +1, 0, 0, Lydian_range32, 2 }, + { "M", +1, M_range16, 186, M_range32, 94 }, + { "Mahajani", +1, 0, 0, Mahajani_range32, 1 }, + { "Makasar", +1, 0, 0, Makasar_range32, 1 }, + { "Malayalam", +1, Malayalam_range16, 8, 0, 0 }, + { "Mandaic", +1, Mandaic_range16, 2, 0, 0 }, + { "Manichaean", +1, 0, 0, Manichaean_range32, 2 }, + { "Marchen", +1, 0, 0, Marchen_range32, 3 }, + { "Masaram_Gondi", +1, 0, 0, Masaram_Gondi_range32, 7 }, + { "Mc", +1, Mc_range16, 109, Mc_range32, 59 }, + { "Me", +1, Me_range16, 5, 0, 0 }, + { "Medefaidrin", +1, 0, 0, Medefaidrin_range32, 1 }, + { "Meetei_Mayek", +1, Meetei_Mayek_range16, 3, 0, 0 }, + { "Mende_Kikakui", +1, 0, 0, Mende_Kikakui_range32, 2 }, + { "Meroitic_Cursive", +1, 0, 0, Meroitic_Cursive_range32, 3 }, + { "Meroitic_Hieroglyphs", +1, 0, 0, Meroitic_Hieroglyphs_range32, 1 }, + { "Miao", +1, 0, 0, Miao_range32, 3 }, + { "Mn", +1, Mn_range16, 207, Mn_range32, 111 }, + { "Modi", +1, 0, 0, Modi_range32, 2 }, + { "Mongolian", +1, Mongolian_range16, 6, Mongolian_range32, 1 }, + { "Mro", +1, 0, 0, Mro_range32, 3 }, + { "Multani", +1, 0, 0, Multani_range32, 5 }, + { "Myanmar", +1, Myanmar_range16, 3, 0, 0 }, + { "N", +1, N_range16, 67, N_range32, 63 }, + { "Nabataean", +1, 0, 0, Nabataean_range32, 2 }, + { "Nandinagari", +1, 0, 0, Nandinagari_range32, 3 }, + { "Nd", +1, Nd_range16, 37, Nd_range32, 22 }, + { "New_Tai_Lue", +1, New_Tai_Lue_range16, 4, 0, 0 }, + { "Newa", +1, 0, 0, Newa_range32, 3 }, + { "Nko", +1, Nko_range16, 2, 0, 0 }, + { "Nl", +1, Nl_range16, 7, Nl_range32, 5 }, + { "No", +1, No_range16, 29, No_range32, 41 }, + { "Nushu", +1, 0, 0, Nushu_range32, 2 }, + { "Nyiakeng_Puachue_Hmong", +1, 0, 0, Nyiakeng_Puachue_Hmong_range32, 4 }, + { "Ogham", +1, Ogham_range16, 1, 0, 0 }, + { "Ol_Chiki", +1, Ol_Chiki_range16, 1, 0, 0 }, + { "Old_Hungarian", +1, 0, 0, Old_Hungarian_range32, 3 }, + { "Old_Italic", +1, 0, 0, Old_Italic_range32, 2 }, + { "Old_North_Arabian", +1, 0, 0, Old_North_Arabian_range32, 1 }, + { "Old_Permic", +1, 0, 0, Old_Permic_range32, 1 }, + { "Old_Persian", +1, 0, 0, Old_Persian_range32, 2 }, + { "Old_Sogdian", +1, 0, 0, Old_Sogdian_range32, 1 }, + { "Old_South_Arabian", +1, 0, 0, Old_South_Arabian_range32, 1 }, + { "Old_Turkic", +1, 0, 0, Old_Turkic_range32, 1 }, + { "Oriya", +1, Oriya_range16, 14, 0, 0 }, + { "Osage", +1, 0, 0, Osage_range32, 2 }, + { "Osmanya", +1, 0, 0, Osmanya_range32, 2 }, + { "P", +1, P_range16, 131, P_range32, 51 }, + { "Pahawh_Hmong", +1, 0, 0, Pahawh_Hmong_range32, 5 }, + { "Palmyrene", +1, 0, 0, Palmyrene_range32, 1 }, + { "Pau_Cin_Hau", +1, 0, 0, Pau_Cin_Hau_range32, 1 }, + { "Pc", +1, Pc_range16, 6, 0, 0 }, + { "Pd", +1, Pd_range16, 17, 0, 0 }, + { "Pe", +1, Pe_range16, 72, 0, 0 }, + { "Pf", +1, Pf_range16, 10, 0, 0 }, + { "Phags_Pa", +1, Phags_Pa_range16, 1, 0, 0 }, + { "Phoenician", +1, 0, 0, Phoenician_range32, 2 }, + { "Pi", +1, Pi_range16, 11, 0, 0 }, + { "Po", +1, Po_range16, 128, Po_range32, 51 }, + { "Ps", +1, Ps_range16, 75, 0, 0 }, + { "Psalter_Pahlavi", +1, 0, 0, Psalter_Pahlavi_range32, 3 }, + { "Rejang", +1, Rejang_range16, 2, 0, 0 }, + { "Runic", +1, Runic_range16, 2, 0, 0 }, + { "S", +1, S_range16, 146, S_range32, 80 }, + { "Samaritan", +1, Samaritan_range16, 2, 0, 0 }, + { "Saurashtra", +1, Saurashtra_range16, 2, 0, 0 }, + { "Sc", +1, Sc_range16, 18, Sc_range32, 3 }, + { "Sharada", +1, 0, 0, Sharada_range32, 2 }, + { "Shavian", +1, 0, 0, Shavian_range32, 1 }, + { "Siddham", +1, 0, 0, Siddham_range32, 2 }, + { "SignWriting", +1, 0, 0, SignWriting_range32, 3 }, + { "Sinhala", +1, Sinhala_range16, 12, Sinhala_range32, 1 }, + { "Sk", +1, Sk_range16, 28, Sk_range32, 1 }, + { "Sm", +1, Sm_range16, 53, Sm_range32, 11 }, + { "So", +1, So_range16, 111, So_range32, 69 }, + { "Sogdian", +1, 0, 0, Sogdian_range32, 1 }, + { "Sora_Sompeng", +1, 0, 0, Sora_Sompeng_range32, 2 }, + { "Soyombo", +1, 0, 0, Soyombo_range32, 1 }, + { "Sundanese", +1, Sundanese_range16, 2, 0, 0 }, + { "Syloti_Nagri", +1, Syloti_Nagri_range16, 1, 0, 0 }, + { "Syriac", +1, Syriac_range16, 4, 0, 0 }, + { "Tagalog", +1, Tagalog_range16, 2, 0, 0 }, + { "Tagbanwa", +1, Tagbanwa_range16, 3, 0, 0 }, + { "Tai_Le", +1, Tai_Le_range16, 2, 0, 0 }, + { "Tai_Tham", +1, Tai_Tham_range16, 5, 0, 0 }, + { "Tai_Viet", +1, Tai_Viet_range16, 2, 0, 0 }, + { "Takri", +1, 0, 0, Takri_range32, 2 }, + { "Tamil", +1, Tamil_range16, 16, Tamil_range32, 2 }, + { "Tangut", +1, 0, 0, Tangut_range32, 3 }, + { "Telugu", +1, Telugu_range16, 12, 0, 0 }, + { "Thaana", +1, Thaana_range16, 1, 0, 0 }, + { "Thai", +1, Thai_range16, 2, 0, 0 }, + { "Tibetan", +1, Tibetan_range16, 7, 0, 0 }, + { "Tifinagh", +1, Tifinagh_range16, 3, 0, 0 }, + { "Tirhuta", +1, 0, 0, Tirhuta_range32, 2 }, + { "Ugaritic", +1, 0, 0, Ugaritic_range32, 2 }, + { "Vai", +1, Vai_range16, 1, 0, 0 }, + { "Wancho", +1, 0, 0, Wancho_range32, 2 }, + { "Warang_Citi", +1, 0, 0, Warang_Citi_range32, 2 }, + { "Yi", +1, Yi_range16, 2, 0, 0 }, + { "Z", +1, Z_range16, 8, 0, 0 }, + { "Zanabazar_Square", +1, 0, 0, Zanabazar_Square_range32, 1 }, + { "Zl", +1, Zl_range16, 1, 0, 0 }, + { "Zp", +1, Zp_range16, 1, 0, 0 }, + { "Zs", +1, Zs_range16, 7, 0, 0 }, +}; +const int num_unicode_groups = 188; + + +} // namespace re2 + + diff --git a/extern/re2/re2/unicode_groups.h b/extern/re2/re2/unicode_groups.h new file mode 100644 index 0000000000..75f55daa61 --- /dev/null +++ b/extern/re2/re2/unicode_groups.h @@ -0,0 +1,67 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_UNICODE_GROUPS_H_ +#define RE2_UNICODE_GROUPS_H_ + +// Unicode character groups. + +// The codes get split into ranges of 16-bit codes +// and ranges of 32-bit codes. It would be simpler +// to use only 32-bit ranges, but these tables are large +// enough to warrant extra care. +// +// Using just 32-bit ranges gives 27 kB of data. +// Adding 16-bit ranges gives 18 kB of data. +// Adding an extra table of 16-bit singletons would reduce +// to 16.5 kB of data but make the data harder to use; +// we don't bother. + +#include + +#include "util/util.h" +#include "util/utf.h" + +namespace re2 { + +struct URange16 +{ + uint16_t lo; + uint16_t hi; +}; + +struct URange32 +{ + Rune lo; + Rune hi; +}; + +struct UGroup +{ + const char *name; + int sign; // +1 for [abc], -1 for [^abc] + const URange16 *r16; + int nr16; + const URange32 *r32; + int nr32; +}; + +// Named by property or script name (e.g., "Nd", "N", "Han"). +// Negated groups are not included. +extern const UGroup unicode_groups[]; +extern const int num_unicode_groups; + +// Named by POSIX name (e.g., "[:alpha:]", "[:^lower:]"). +// Negated groups are included. +extern const UGroup posix_groups[]; +extern const int num_posix_groups; + +// Named by Perl name (e.g., "\\d", "\\D"). +// Negated groups are included. +extern const UGroup perl_groups[]; +extern const int num_perl_groups; + +} // namespace re2 + +#endif // RE2_UNICODE_GROUPS_H_ diff --git a/extern/re2/re2/walker-inl.h b/extern/re2/re2/walker-inl.h new file mode 100644 index 0000000000..032b8ac7db --- /dev/null +++ b/extern/re2/re2/walker-inl.h @@ -0,0 +1,248 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef RE2_WALKER_INL_H_ +#define RE2_WALKER_INL_H_ + +// Helper class for traversing Regexps without recursion. +// Clients should declare their own subclasses that override +// the PreVisit and PostVisit methods, which are called before +// and after visiting the subexpressions. + +// Not quite the Visitor pattern, because (among other things) +// the Visitor pattern is recursive. + +#include + +#include "util/logging.h" +#include "re2/regexp.h" + +namespace re2 { + +template struct WalkState; + +template class Regexp::Walker { + public: + Walker(); + virtual ~Walker(); + + // Virtual method called before visiting re's children. + // PreVisit passes ownership of its return value to its caller. + // The Arg* that PreVisit returns will be passed to PostVisit as pre_arg + // and passed to the child PreVisits and PostVisits as parent_arg. + // At the top-most Regexp, parent_arg is arg passed to walk. + // If PreVisit sets *stop to true, the walk does not recurse + // into the children. Instead it behaves as though the return + // value from PreVisit is the return value from PostVisit. + // The default PreVisit returns parent_arg. + virtual T PreVisit(Regexp* re, T parent_arg, bool* stop); + + // Virtual method called after visiting re's children. + // The pre_arg is the T that PreVisit returned. + // The child_args is a vector of the T that the child PostVisits returned. + // PostVisit takes ownership of pre_arg. + // PostVisit takes ownership of the Ts + // in *child_args, but not the vector itself. + // PostVisit passes ownership of its return value + // to its caller. + // The default PostVisit simply returns pre_arg. + virtual T PostVisit(Regexp* re, T parent_arg, T pre_arg, + T* child_args, int nchild_args); + + // Virtual method called to copy a T, + // when Walk notices that more than one child is the same re. + virtual T Copy(T arg); + + // Virtual method called to do a "quick visit" of the re, + // but not its children. Only called once the visit budget + // has been used up and we're trying to abort the walk + // as quickly as possible. Should return a value that + // makes sense for the parent PostVisits still to be run. + // This function is (hopefully) only called by + // WalkExponential, but must be implemented by all clients, + // just in case. + virtual T ShortVisit(Regexp* re, T parent_arg) = 0; + + // Walks over a regular expression. + // Top_arg is passed as parent_arg to PreVisit and PostVisit of re. + // Returns the T returned by PostVisit on re. + T Walk(Regexp* re, T top_arg); + + // Like Walk, but doesn't use Copy. This can lead to + // exponential runtimes on cross-linked Regexps like the + // ones generated by Simplify. To help limit this, + // at most max_visits nodes will be visited and then + // the walk will be cut off early. + // If the walk *is* cut off early, ShortVisit(re) + // will be called on regexps that cannot be fully + // visited rather than calling PreVisit/PostVisit. + T WalkExponential(Regexp* re, T top_arg, int max_visits); + + // Clears the stack. Should never be necessary, since + // Walk always enters and exits with an empty stack. + // Logs DFATAL if stack is not already clear. + void Reset(); + + // Returns whether walk was cut off. + bool stopped_early() { return stopped_early_; } + + private: + // Walk state for the entire traversal. + std::stack >* stack_; + bool stopped_early_; + int max_visits_; + + T WalkInternal(Regexp* re, T top_arg, bool use_copy); + + Walker(const Walker&) = delete; + Walker& operator=(const Walker&) = delete; +}; + +template T Regexp::Walker::PreVisit(Regexp* re, + T parent_arg, + bool* stop) { + return parent_arg; +} + +template T Regexp::Walker::PostVisit(Regexp* re, + T parent_arg, + T pre_arg, + T* child_args, + int nchild_args) { + return pre_arg; +} + +template T Regexp::Walker::Copy(T arg) { + return arg; +} + +// State about a single level in the traversal. +template struct WalkState { + WalkState(Regexp* re, T parent) + : re(re), + n(-1), + parent_arg(parent), + child_args(NULL) { } + + Regexp* re; // The regexp + int n; // The index of the next child to process; -1 means need to PreVisit + T parent_arg; // Accumulated arguments. + T pre_arg; + T child_arg; // One-element buffer for child_args. + T* child_args; +}; + +template Regexp::Walker::Walker() { + stack_ = new std::stack >; + stopped_early_ = false; +} + +template Regexp::Walker::~Walker() { + Reset(); + delete stack_; +} + +// Clears the stack. Should never be necessary, since +// Walk always enters and exits with an empty stack. +// Logs DFATAL if stack is not already clear. +template void Regexp::Walker::Reset() { + if (stack_ && stack_->size() > 0) { + LOG(DFATAL) << "Stack not empty."; + while (stack_->size() > 0) { + delete stack_->top().child_args; + stack_->pop(); + } + } +} + +template T Regexp::Walker::WalkInternal(Regexp* re, T top_arg, + bool use_copy) { + Reset(); + + if (re == NULL) { + LOG(DFATAL) << "Walk NULL"; + return top_arg; + } + + stack_->push(WalkState(re, top_arg)); + + WalkState* s; + for (;;) { + T t; + s = &stack_->top(); + Regexp* re = s->re; + switch (s->n) { + case -1: { + if (--max_visits_ < 0) { + stopped_early_ = true; + t = ShortVisit(re, s->parent_arg); + break; + } + bool stop = false; + s->pre_arg = PreVisit(re, s->parent_arg, &stop); + if (stop) { + t = s->pre_arg; + break; + } + s->n = 0; + s->child_args = NULL; + if (re->nsub_ == 1) + s->child_args = &s->child_arg; + else if (re->nsub_ > 1) + s->child_args = new T[re->nsub_]; + FALLTHROUGH_INTENDED; + } + default: { + if (re->nsub_ > 0) { + Regexp** sub = re->sub(); + if (s->n < re->nsub_) { + if (use_copy && s->n > 0 && sub[s->n - 1] == sub[s->n]) { + s->child_args[s->n] = Copy(s->child_args[s->n - 1]); + s->n++; + } else { + stack_->push(WalkState(sub[s->n], s->pre_arg)); + } + continue; + } + } + + t = PostVisit(re, s->parent_arg, s->pre_arg, s->child_args, s->n); + if (re->nsub_ > 1) + delete[] s->child_args; + break; + } + } + + // We've finished stack_->top(). + // Update next guy down. + stack_->pop(); + if (stack_->size() == 0) + return t; + s = &stack_->top(); + if (s->child_args != NULL) + s->child_args[s->n] = t; + else + s->child_arg = t; + s->n++; + } +} + +template T Regexp::Walker::Walk(Regexp* re, T top_arg) { + // Without the exponential walking behavior, + // this budget should be more than enough for any + // regexp, and yet not enough to get us in trouble + // as far as CPU time. + max_visits_ = 1000000; + return WalkInternal(re, top_arg, true); +} + +template T Regexp::Walker::WalkExponential(Regexp* re, T top_arg, + int max_visits) { + max_visits_ = max_visits; + return WalkInternal(re, top_arg, false); +} + +} // namespace re2 + +#endif // RE2_WALKER_INL_H_ diff --git a/extern/re2/re2_test.bzl b/extern/re2/re2_test.bzl new file mode 100644 index 0000000000..c0eb654196 --- /dev/null +++ b/extern/re2/re2_test.bzl @@ -0,0 +1,12 @@ +# Copyright 2009 The RE2 Authors. All Rights Reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Defines a Bazel macro that instantiates a native cc_test rule for an RE2 test. +def re2_test(name, deps=[], size="medium"): + native.cc_test( + name=name, + srcs=["re2/testing/%s.cc" % (name)], + deps=[":test"] + deps, + size=size, + ) diff --git a/extern/re2/runtests b/extern/re2/runtests new file mode 100644 index 0000000000..94584a660d --- /dev/null +++ b/extern/re2/runtests @@ -0,0 +1,33 @@ +#!/usr/bin/env sh + +# System Integrity Protection on Darwin complicated these matters somewhat. +# See https://github.com/google/re2/issues/175 for details. +if [ "x$1" = "x-shared-library-path" ]; then + if [ "x$(uname)" = "xDarwin" ]; then + DYLD_LIBRARY_PATH="$2:$DYLD_LIBRARY_PATH" + export DYLD_LIBRARY_PATH + else + LD_LIBRARY_PATH="$2:$LD_LIBRARY_PATH" + export LD_LIBRARY_PATH + fi + shift 2 +fi + +success=true +for i; do + printf "%-40s" $i + if $($i >$i.log 2>&1) 2>/dev/null; then + echo PASS + else + echo FAIL';' output in $i.log + success=false + fi +done + +if $success; then + echo 'ALL TESTS PASSED.' + exit 0 +else + echo 'TESTS FAILED.' + exit 1 +fi diff --git a/extern/re2/testinstall.cc b/extern/re2/testinstall.cc new file mode 100644 index 0000000000..47db4e68cc --- /dev/null +++ b/extern/re2/testinstall.cc @@ -0,0 +1,24 @@ +// Copyright 2008 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include +#include + +int main(void) { + re2::FilteredRE2 f; + int id; + f.Add("a.*b.*c", RE2::DefaultOptions, &id); + std::vector v; + f.Compile(&v); + std::vector ids; + f.FirstMatch("abbccc", ids); + + if(RE2::FullMatch("axbyc", "a.*b.*c")) { + printf("PASS\n"); + return 0; + } + printf("FAIL\n"); + return 2; +} diff --git a/extern/re2/util/benchmark.cc b/extern/re2/util/benchmark.cc new file mode 100644 index 0000000000..144f550171 --- /dev/null +++ b/extern/re2/util/benchmark.cc @@ -0,0 +1,161 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include +#include +#include +#include +#include + +#include "util/util.h" +#include "util/flags.h" +#include "util/benchmark.h" +#include "re2/re2.h" + +DEFINE_string(test_tmpdir, "/var/tmp", "temp directory"); + +#ifdef _WIN32 +#define snprintf _snprintf +#endif + +using testing::Benchmark; + +static Benchmark* benchmarks[10000]; +static int nbenchmarks; + +void Benchmark::Register() { + benchmarks[nbenchmarks] = this; + if(lo < 1) + lo = 1; + if(hi < lo) + hi = lo; + nbenchmarks++; +} + +static int64_t nsec() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +static int64_t bytes; +static int64_t ns; +static int64_t t0; +static int64_t items; + +void SetBenchmarkBytesProcessed(int64_t x) { + bytes = x; +} + +void StopBenchmarkTiming() { + if(t0 != 0) + ns += nsec() - t0; + t0 = 0; +} + +void StartBenchmarkTiming() { + if(t0 == 0) + t0 = nsec(); +} + +void SetBenchmarkItemsProcessed(int n) { + items = n; +} + +void BenchmarkMemoryUsage() { + // TODO(rsc): Implement. +} + +int NumCPUs() { + return static_cast(std::thread::hardware_concurrency()); +} + +static void runN(Benchmark *b, int n, int siz) { + bytes = 0; + items = 0; + ns = 0; + t0 = nsec(); + if(b->fn) + b->fn(n); + else if(b->fnr) + b->fnr(n, siz); + else { + fprintf(stderr, "%s: missing function\n", b->name); + abort(); + } + if(t0 != 0) + ns += nsec() - t0; +} + +static int round(int n) { + int base = 1; + + while(base*10 < n) + base *= 10; + if(n < 2*base) + return 2*base; + if(n < 5*base) + return 5*base; + return 10*base; +} + +void RunBench(Benchmark* b, int nthread, int siz) { + int n, last; + + // TODO(rsc): Threaded benchmarks. + if(nthread != 1) + return; + + // run once in case it's expensive + n = 1; + runN(b, n, siz); + while(ns < (int)1e9 && n < (int)1e9) { + last = n; + if(ns/n == 0) + n = (int)1e9; + else + n = (int)1e9 / static_cast(ns/n); + + n = std::max(last+1, std::min(n+n/2, 100*last)); + n = round(n); + runN(b, n, siz); + } + + char mb[100]; + char suf[100]; + mb[0] = '\0'; + suf[0] = '\0'; + if(ns > 0 && bytes > 0) + snprintf(mb, sizeof mb, "\t%7.2f MB/s", ((double)bytes/1e6)/((double)ns/1e9)); + if(b->fnr || b->lo != b->hi) { + if(siz >= (1<<20)) + snprintf(suf, sizeof suf, "/%dM", siz/(1<<20)); + else if(siz >= (1<<10)) + snprintf(suf, sizeof suf, "/%dK", siz/(1<<10)); + else + snprintf(suf, sizeof suf, "/%d", siz); + } + printf("%s%s\t%8lld\t%10lld ns/op%s\n", b->name, suf, (long long)n, (long long)ns/n, mb); + fflush(stdout); +} + +static int match(const char* name, int argc, const char** argv) { + if(argc == 1) + return 1; + for(int i = 1; i < argc; i++) + if(RE2::PartialMatch(name, argv[i])) + return 1; + return 0; +} + +int main(int argc, const char** argv) { + for(int i = 0; i < nbenchmarks; i++) { + Benchmark* b = benchmarks[i]; + if(match(b->name, argc, argv)) + for(int j = b->threadlo; j <= b->threadhi; j++) + for(int k = std::max(b->lo, 1); k <= std::max(b->hi, 1); k<<=1) + RunBench(b, j, k); + } +} + diff --git a/extern/re2/util/benchmark.h b/extern/re2/util/benchmark.h new file mode 100644 index 0000000000..fba30b9cba --- /dev/null +++ b/extern/re2/util/benchmark.h @@ -0,0 +1,43 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef UTIL_BENCHMARK_H_ +#define UTIL_BENCHMARK_H_ + +#include + +namespace testing { +struct Benchmark { + const char* name; + void (*fn)(int); + void (*fnr)(int, int); + int lo; + int hi; + int threadlo; + int threadhi; + + void Register(); + Benchmark(const char* name, void (*f)(int)) { Clear(name); fn = f; Register(); } + Benchmark(const char* name, void (*f)(int, int), int l, int h) { Clear(name); fnr = f; lo = l; hi = h; Register(); } + void Clear(const char* n) { name = n; fn = 0; fnr = 0; lo = 0; hi = 0; threadlo = 0; threadhi = 0; } + Benchmark* ThreadRange(int lo, int hi) { threadlo = lo; threadhi = hi; return this; } +}; +} // namespace testing + +void SetBenchmarkBytesProcessed(int64_t); +void StopBenchmarkTiming(); +void StartBenchmarkTiming(); +void BenchmarkMemoryUsage(); +void SetBenchmarkItemsProcessed(int); + +int NumCPUs(); + +#define BENCHMARK(f) \ + ::testing::Benchmark* _benchmark_##f = (new ::testing::Benchmark(#f, f)) + +#define BENCHMARK_RANGE(f, lo, hi) \ + ::testing::Benchmark* _benchmark_##f = \ + (new ::testing::Benchmark(#f, f, lo, hi)) + +#endif // UTIL_BENCHMARK_H_ diff --git a/extern/re2/util/flags.h b/extern/re2/util/flags.h new file mode 100644 index 0000000000..e0f1f420bc --- /dev/null +++ b/extern/re2/util/flags.h @@ -0,0 +1,29 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef UTIL_FLAGS_H_ +#define UTIL_FLAGS_H_ + +// Simplified version of Google's command line flags. +// Does not support parsing the command line. +// If you want to do that, see +// https://gflags.github.io/gflags/ + +#include + +#define DEFINE_flag(type, name, deflt, desc) \ + namespace re2 { type FLAGS_##name = deflt; } + +#define DECLARE_flag(type, name) \ + namespace re2 { extern type FLAGS_##name; } + +#define DEFINE_bool(name, deflt, desc) DEFINE_flag(bool, name, deflt, desc) +#define DEFINE_int32(name, deflt, desc) DEFINE_flag(int32_t, name, deflt, desc) +#define DEFINE_string(name, deflt, desc) DEFINE_flag(std::string, name, deflt, desc) + +#define DECLARE_bool(name) DECLARE_flag(bool, name) +#define DECLARE_int32(name) DECLARE_flag(int32_t, name) +#define DECLARE_string(name) DECLARE_flag(std::string, name) + +#endif // UTIL_FLAGS_H_ diff --git a/extern/re2/util/fuzz.cc b/extern/re2/util/fuzz.cc new file mode 100644 index 0000000000..9cac1185ac --- /dev/null +++ b/extern/re2/util/fuzz.cc @@ -0,0 +1,21 @@ +// Copyright 2016 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include +#include + +// Entry point for libFuzzer. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size); + +int main(int argc, char** argv) { + uint8_t data[32]; + for (int i = 0; i < 32; i++) { + for (int j = 0; j < 32; j++) { + data[j] = random() & 0xFF; + } + LLVMFuzzerTestOneInput(data, 32); + } + return 0; +} diff --git a/extern/re2/util/logging.h b/extern/re2/util/logging.h new file mode 100644 index 0000000000..5b2217f29c --- /dev/null +++ b/extern/re2/util/logging.h @@ -0,0 +1,109 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef UTIL_LOGGING_H_ +#define UTIL_LOGGING_H_ + +// Simplified version of Google's logging. + +#include +#include +#include +#include +#include + +#include "util/util.h" + +// Debug-only checking. +#define DCHECK(condition) assert(condition) +#define DCHECK_EQ(val1, val2) assert((val1) == (val2)) +#define DCHECK_NE(val1, val2) assert((val1) != (val2)) +#define DCHECK_LE(val1, val2) assert((val1) <= (val2)) +#define DCHECK_LT(val1, val2) assert((val1) < (val2)) +#define DCHECK_GE(val1, val2) assert((val1) >= (val2)) +#define DCHECK_GT(val1, val2) assert((val1) > (val2)) + +// Always-on checking +#define CHECK(x) if(x){}else LogMessageFatal(__FILE__, __LINE__).stream() << "Check failed: " #x +#define CHECK_LT(x, y) CHECK((x) < (y)) +#define CHECK_GT(x, y) CHECK((x) > (y)) +#define CHECK_LE(x, y) CHECK((x) <= (y)) +#define CHECK_GE(x, y) CHECK((x) >= (y)) +#define CHECK_EQ(x, y) CHECK((x) == (y)) +#define CHECK_NE(x, y) CHECK((x) != (y)) + +#define LOG_INFO LogMessage(__FILE__, __LINE__) +#define LOG_WARNING LogMessage(__FILE__, __LINE__) +#define LOG_ERROR LogMessage(__FILE__, __LINE__) +#define LOG_FATAL LogMessageFatal(__FILE__, __LINE__) +#define LOG_QFATAL LOG_FATAL + +// It seems that one of the Windows header files defines ERROR as 0. +#ifdef _WIN32 +#define LOG_0 LOG_INFO +#endif + +#ifdef NDEBUG +#define LOG_DFATAL LOG_ERROR +#else +#define LOG_DFATAL LOG_FATAL +#endif + +#define LOG(severity) LOG_ ## severity.stream() + +#define VLOG(x) if((x)>0){}else LOG_INFO.stream() + +class LogMessage { + public: + LogMessage(const char* file, int line) + : flushed_(false) { + stream() << file << ":" << line << ": "; + } + void Flush() { + stream() << "\n"; + std::string s = str_.str(); + size_t n = s.size(); + if (fwrite(s.data(), 1, n, stderr) < n) {} // shut up gcc + flushed_ = true; + } + ~LogMessage() { + if (!flushed_) { + Flush(); + } + } + std::ostream& stream() { return str_; } + + private: + bool flushed_; + std::ostringstream str_; + + LogMessage(const LogMessage&) = delete; + LogMessage& operator=(const LogMessage&) = delete; +}; + +// Silence "destructor never returns" warning for ~LogMessageFatal(). +// Since this is a header file, push and then pop to limit the scope. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4722) +#endif + +class LogMessageFatal : public LogMessage { + public: + LogMessageFatal(const char* file, int line) + : LogMessage(file, line) {} + ATTRIBUTE_NORETURN ~LogMessageFatal() { + Flush(); + abort(); + } + private: + LogMessageFatal(const LogMessageFatal&) = delete; + LogMessageFatal& operator=(const LogMessageFatal&) = delete; +}; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#endif // UTIL_LOGGING_H_ diff --git a/extern/re2/util/mix.h b/extern/re2/util/mix.h new file mode 100644 index 0000000000..d85c172ab0 --- /dev/null +++ b/extern/re2/util/mix.h @@ -0,0 +1,41 @@ +// Copyright 2016 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef UTIL_MIX_H_ +#define UTIL_MIX_H_ + +#include +#include + +namespace re2 { + +// Silence "truncation of constant value" warning for kMul in 32-bit mode. +// Since this is a header file, push and then pop to limit the scope. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4309) +#endif + +class HashMix { + public: + HashMix() : hash_(1) {} + explicit HashMix(size_t val) : hash_(val + 83) {} + void Mix(size_t val) { + static const size_t kMul = static_cast(0xdc3eb94af8ab4c93ULL); + hash_ *= kMul; + hash_ = ((hash_ << 19) | + (hash_ >> (std::numeric_limits::digits - 19))) + val; + } + size_t get() const { return hash_; } + private: + size_t hash_; +}; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +} // namespace re2 + +#endif // UTIL_MIX_H_ diff --git a/extern/re2/util/mutex.h b/extern/re2/util/mutex.h new file mode 100644 index 0000000000..9c49158048 --- /dev/null +++ b/extern/re2/util/mutex.h @@ -0,0 +1,131 @@ +// Copyright 2007 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef UTIL_MUTEX_H_ +#define UTIL_MUTEX_H_ + +/* + * A simple mutex wrapper, supporting locks and read-write locks. + * You should assume the locks are *not* re-entrant. + */ + +#if !defined(_WIN32) +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 200809L +#endif +#include +#if defined(_POSIX_READER_WRITER_LOCKS) && _POSIX_READER_WRITER_LOCKS > 0 +#define MUTEX_IS_PTHREAD_RWLOCK +#endif +#endif + +#if defined(MUTEX_IS_PTHREAD_RWLOCK) +#include +#include +typedef pthread_rwlock_t MutexType; +#else +#include +typedef std::mutex MutexType; +#endif + +namespace re2 { + +class Mutex { + public: + inline Mutex(); + inline ~Mutex(); + inline void Lock(); // Block if needed until free then acquire exclusively + inline void Unlock(); // Release a lock acquired via Lock() + // Note that on systems that don't support read-write locks, these may + // be implemented as synonyms to Lock() and Unlock(). So you can use + // these for efficiency, but don't use them anyplace where being able + // to do shared reads is necessary to avoid deadlock. + inline void ReaderLock(); // Block until free or shared then acquire a share + inline void ReaderUnlock(); // Release a read share of this Mutex + inline void WriterLock() { Lock(); } // Acquire an exclusive lock + inline void WriterUnlock() { Unlock(); } // Release a lock from WriterLock() + + private: + MutexType mutex_; + + // Catch the error of writing Mutex when intending MutexLock. + Mutex(Mutex *ignored); + + Mutex(const Mutex&) = delete; + Mutex& operator=(const Mutex&) = delete; +}; + +#if defined(MUTEX_IS_PTHREAD_RWLOCK) + +#define SAFE_PTHREAD(fncall) \ + do { \ + if ((fncall) != 0) abort(); \ + } while (0) + +Mutex::Mutex() { SAFE_PTHREAD(pthread_rwlock_init(&mutex_, NULL)); } +Mutex::~Mutex() { SAFE_PTHREAD(pthread_rwlock_destroy(&mutex_)); } +void Mutex::Lock() { SAFE_PTHREAD(pthread_rwlock_wrlock(&mutex_)); } +void Mutex::Unlock() { SAFE_PTHREAD(pthread_rwlock_unlock(&mutex_)); } +void Mutex::ReaderLock() { SAFE_PTHREAD(pthread_rwlock_rdlock(&mutex_)); } +void Mutex::ReaderUnlock() { SAFE_PTHREAD(pthread_rwlock_unlock(&mutex_)); } + +#undef SAFE_PTHREAD + +#else + +Mutex::Mutex() { } +Mutex::~Mutex() { } +void Mutex::Lock() { mutex_.lock(); } +void Mutex::Unlock() { mutex_.unlock(); } +void Mutex::ReaderLock() { Lock(); } // C++11 doesn't have std::shared_mutex. +void Mutex::ReaderUnlock() { Unlock(); } + +#endif + +// -------------------------------------------------------------------------- +// Some helper classes + +// MutexLock(mu) acquires mu when constructed and releases it when destroyed. +class MutexLock { + public: + explicit MutexLock(Mutex *mu) : mu_(mu) { mu_->Lock(); } + ~MutexLock() { mu_->Unlock(); } + private: + Mutex * const mu_; + + MutexLock(const MutexLock&) = delete; + MutexLock& operator=(const MutexLock&) = delete; +}; + +// ReaderMutexLock and WriterMutexLock do the same, for rwlocks +class ReaderMutexLock { + public: + explicit ReaderMutexLock(Mutex *mu) : mu_(mu) { mu_->ReaderLock(); } + ~ReaderMutexLock() { mu_->ReaderUnlock(); } + private: + Mutex * const mu_; + + ReaderMutexLock(const ReaderMutexLock&) = delete; + ReaderMutexLock& operator=(const ReaderMutexLock&) = delete; +}; + +class WriterMutexLock { + public: + explicit WriterMutexLock(Mutex *mu) : mu_(mu) { mu_->WriterLock(); } + ~WriterMutexLock() { mu_->WriterUnlock(); } + private: + Mutex * const mu_; + + WriterMutexLock(const WriterMutexLock&) = delete; + WriterMutexLock& operator=(const WriterMutexLock&) = delete; +}; + +// Catch bug where variable name is omitted, e.g. MutexLock (&mu); +#define MutexLock(x) static_assert(false, "MutexLock declaration missing variable name") +#define ReaderMutexLock(x) static_assert(false, "ReaderMutexLock declaration missing variable name") +#define WriterMutexLock(x) static_assert(false, "WriterMutexLock declaration missing variable name") + +} // namespace re2 + +#endif // UTIL_MUTEX_H_ diff --git a/extern/re2/util/pcre.cc b/extern/re2/util/pcre.cc new file mode 100644 index 0000000000..5983c9f526 --- /dev/null +++ b/extern/re2/util/pcre.cc @@ -0,0 +1,1051 @@ +// Copyright 2003-2009 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This is a variant of PCRE's pcrecpp.cc, originally written at Google. +// The main changes are the addition of the HitLimit method and +// compilation as PCRE in namespace re2. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util/util.h" +#include "util/flags.h" +#include "util/logging.h" +#include "util/pcre.h" +#include "util/strutil.h" + +// Silence warnings about the wacky formatting in the operator() functions. +// Note that we test for Clang first because it defines __GNUC__ as well. +#if defined(__clang__) +#elif defined(__GNUC__) && __GNUC__ >= 6 +#pragma GCC diagnostic ignored "-Wmisleading-indentation" +#endif + +#define PCREPORT(level) LOG(level) + +// Default PCRE limits. +// Defaults chosen to allow a plausible amount of CPU and +// not exceed main thread stacks. Note that other threads +// often have smaller stacks, and therefore tightening +// regexp_stack_limit may frequently be necessary. +DEFINE_int32(regexp_stack_limit, 256<<10, "default PCRE stack limit (bytes)"); +DEFINE_int32(regexp_match_limit, 1000000, + "default PCRE match limit (function calls)"); + +#ifndef USEPCRE + +// Fake just enough of the PCRE API to allow this file to build. :) + +struct pcre_extra { + int flags; + int match_limit; + int match_limit_recursion; +}; + +#define PCRE_EXTRA_MATCH_LIMIT 0 +#define PCRE_EXTRA_MATCH_LIMIT_RECURSION 0 +#define PCRE_ANCHORED 0 +#define PCRE_NOTEMPTY 0 +#define PCRE_ERROR_NOMATCH 1 +#define PCRE_ERROR_MATCHLIMIT 2 +#define PCRE_ERROR_RECURSIONLIMIT 3 +#define PCRE_INFO_CAPTURECOUNT 0 + +void pcre_free(void*) { +} + +pcre* pcre_compile(const char*, int, const char**, int*, const unsigned char*) { + return NULL; +} + +int pcre_exec(const pcre*, const pcre_extra*, const char*, int, int, int, int*, int) { + return 0; +} + +int pcre_fullinfo(const pcre*, const pcre_extra*, int, void*) { + return 0; +} + +#endif + +namespace re2 { + +// Maximum number of args we can set +static const int kMaxArgs = 16; +static const int kVecSize = (1 + kMaxArgs) * 3; // results + PCRE workspace + +// Approximate size of a recursive invocation of PCRE's +// internal "match()" frame. This varies depending on the +// compiler and architecture, of course, so the constant is +// just a conservative estimate. To find the exact number, +// run regexp_unittest with --regexp_stack_limit=0 under +// a debugger and look at the frames when it crashes. +// The exact frame size was 656 in production on 2008/02/03. +static const int kPCREFrameSize = 700; + +// Special name for missing C++ arguments. +PCRE::Arg PCRE::no_more_args((void*)NULL); + +const PCRE::PartialMatchFunctor PCRE::PartialMatch = { }; +const PCRE::FullMatchFunctor PCRE::FullMatch = { } ; +const PCRE::ConsumeFunctor PCRE::Consume = { }; +const PCRE::FindAndConsumeFunctor PCRE::FindAndConsume = { }; + +// If a regular expression has no error, its error_ field points here +static const std::string empty_string; + +void PCRE::Init(const char* pattern, Option options, int match_limit, + int stack_limit, bool report_errors) { + pattern_ = pattern; + options_ = options; + match_limit_ = match_limit; + stack_limit_ = stack_limit; + hit_limit_ = false; + error_ = &empty_string; + report_errors_ = report_errors; + re_full_ = NULL; + re_partial_ = NULL; + + if (options & ~(EnabledCompileOptions | EnabledExecOptions)) { + error_ = new std::string("illegal regexp option"); + PCREPORT(ERROR) + << "Error compiling '" << pattern << "': illegal regexp option"; + } else { + re_partial_ = Compile(UNANCHORED); + if (re_partial_ != NULL) { + re_full_ = Compile(ANCHOR_BOTH); + } + } +} + +PCRE::PCRE(const char* pattern) { + Init(pattern, None, 0, 0, true); +} +PCRE::PCRE(const char* pattern, Option option) { + Init(pattern, option, 0, 0, true); +} +PCRE::PCRE(const std::string& pattern) { + Init(pattern.c_str(), None, 0, 0, true); +} +PCRE::PCRE(const std::string& pattern, Option option) { + Init(pattern.c_str(), option, 0, 0, true); +} +PCRE::PCRE(const std::string& pattern, const PCRE_Options& re_option) { + Init(pattern.c_str(), re_option.option(), re_option.match_limit(), + re_option.stack_limit(), re_option.report_errors()); +} + +PCRE::PCRE(const char *pattern, const PCRE_Options& re_option) { + Init(pattern, re_option.option(), re_option.match_limit(), + re_option.stack_limit(), re_option.report_errors()); +} + +PCRE::~PCRE() { + if (re_full_ != NULL) pcre_free(re_full_); + if (re_partial_ != NULL) pcre_free(re_partial_); + if (error_ != &empty_string) delete error_; +} + +pcre* PCRE::Compile(Anchor anchor) { + // Special treatment for anchoring. This is needed because at + // runtime pcre only provides an option for anchoring at the + // beginning of a string. + // + // There are three types of anchoring we want: + // UNANCHORED Compile the original pattern, and use + // a pcre unanchored match. + // ANCHOR_START Compile the original pattern, and use + // a pcre anchored match. + // ANCHOR_BOTH Tack a "\z" to the end of the original pattern + // and use a pcre anchored match. + + const char* error = ""; + int eoffset; + pcre* re; + if (anchor != ANCHOR_BOTH) { + re = pcre_compile(pattern_.c_str(), + (options_ & EnabledCompileOptions), + &error, &eoffset, NULL); + } else { + // Tack a '\z' at the end of PCRE. Parenthesize it first so that + // the '\z' applies to all top-level alternatives in the regexp. + std::string wrapped = "(?:"; // A non-counting grouping operator + wrapped += pattern_; + wrapped += ")\\z"; + re = pcre_compile(wrapped.c_str(), + (options_ & EnabledCompileOptions), + &error, &eoffset, NULL); + } + if (re == NULL) { + if (error_ == &empty_string) error_ = new std::string(error); + PCREPORT(ERROR) << "Error compiling '" << pattern_ << "': " << error; + } + return re; +} + +/***** Convenience interfaces *****/ + +bool PCRE::FullMatchFunctor::operator ()(const StringPiece& text, + const PCRE& re, + const Arg& a0, + const Arg& a1, + const Arg& a2, + const Arg& a3, + const Arg& a4, + const Arg& a5, + const Arg& a6, + const Arg& a7, + const Arg& a8, + const Arg& a9, + const Arg& a10, + const Arg& a11, + const Arg& a12, + const Arg& a13, + const Arg& a14, + const Arg& a15) const { + const Arg* args[kMaxArgs]; + int n = 0; + if (&a0 == &no_more_args) goto done; args[n++] = &a0; + if (&a1 == &no_more_args) goto done; args[n++] = &a1; + if (&a2 == &no_more_args) goto done; args[n++] = &a2; + if (&a3 == &no_more_args) goto done; args[n++] = &a3; + if (&a4 == &no_more_args) goto done; args[n++] = &a4; + if (&a5 == &no_more_args) goto done; args[n++] = &a5; + if (&a6 == &no_more_args) goto done; args[n++] = &a6; + if (&a7 == &no_more_args) goto done; args[n++] = &a7; + if (&a8 == &no_more_args) goto done; args[n++] = &a8; + if (&a9 == &no_more_args) goto done; args[n++] = &a9; + if (&a10 == &no_more_args) goto done; args[n++] = &a10; + if (&a11 == &no_more_args) goto done; args[n++] = &a11; + if (&a12 == &no_more_args) goto done; args[n++] = &a12; + if (&a13 == &no_more_args) goto done; args[n++] = &a13; + if (&a14 == &no_more_args) goto done; args[n++] = &a14; + if (&a15 == &no_more_args) goto done; args[n++] = &a15; +done: + + size_t consumed; + int vec[kVecSize] = {}; + return re.DoMatchImpl(text, ANCHOR_BOTH, &consumed, args, n, vec, kVecSize); +} + +bool PCRE::PartialMatchFunctor::operator ()(const StringPiece& text, + const PCRE& re, + const Arg& a0, + const Arg& a1, + const Arg& a2, + const Arg& a3, + const Arg& a4, + const Arg& a5, + const Arg& a6, + const Arg& a7, + const Arg& a8, + const Arg& a9, + const Arg& a10, + const Arg& a11, + const Arg& a12, + const Arg& a13, + const Arg& a14, + const Arg& a15) const { + const Arg* args[kMaxArgs]; + int n = 0; + if (&a0 == &no_more_args) goto done; args[n++] = &a0; + if (&a1 == &no_more_args) goto done; args[n++] = &a1; + if (&a2 == &no_more_args) goto done; args[n++] = &a2; + if (&a3 == &no_more_args) goto done; args[n++] = &a3; + if (&a4 == &no_more_args) goto done; args[n++] = &a4; + if (&a5 == &no_more_args) goto done; args[n++] = &a5; + if (&a6 == &no_more_args) goto done; args[n++] = &a6; + if (&a7 == &no_more_args) goto done; args[n++] = &a7; + if (&a8 == &no_more_args) goto done; args[n++] = &a8; + if (&a9 == &no_more_args) goto done; args[n++] = &a9; + if (&a10 == &no_more_args) goto done; args[n++] = &a10; + if (&a11 == &no_more_args) goto done; args[n++] = &a11; + if (&a12 == &no_more_args) goto done; args[n++] = &a12; + if (&a13 == &no_more_args) goto done; args[n++] = &a13; + if (&a14 == &no_more_args) goto done; args[n++] = &a14; + if (&a15 == &no_more_args) goto done; args[n++] = &a15; +done: + + size_t consumed; + int vec[kVecSize] = {}; + return re.DoMatchImpl(text, UNANCHORED, &consumed, args, n, vec, kVecSize); +} + +bool PCRE::ConsumeFunctor::operator ()(StringPiece* input, + const PCRE& pattern, + const Arg& a0, + const Arg& a1, + const Arg& a2, + const Arg& a3, + const Arg& a4, + const Arg& a5, + const Arg& a6, + const Arg& a7, + const Arg& a8, + const Arg& a9, + const Arg& a10, + const Arg& a11, + const Arg& a12, + const Arg& a13, + const Arg& a14, + const Arg& a15) const { + const Arg* args[kMaxArgs]; + int n = 0; + if (&a0 == &no_more_args) goto done; args[n++] = &a0; + if (&a1 == &no_more_args) goto done; args[n++] = &a1; + if (&a2 == &no_more_args) goto done; args[n++] = &a2; + if (&a3 == &no_more_args) goto done; args[n++] = &a3; + if (&a4 == &no_more_args) goto done; args[n++] = &a4; + if (&a5 == &no_more_args) goto done; args[n++] = &a5; + if (&a6 == &no_more_args) goto done; args[n++] = &a6; + if (&a7 == &no_more_args) goto done; args[n++] = &a7; + if (&a8 == &no_more_args) goto done; args[n++] = &a8; + if (&a9 == &no_more_args) goto done; args[n++] = &a9; + if (&a10 == &no_more_args) goto done; args[n++] = &a10; + if (&a11 == &no_more_args) goto done; args[n++] = &a11; + if (&a12 == &no_more_args) goto done; args[n++] = &a12; + if (&a13 == &no_more_args) goto done; args[n++] = &a13; + if (&a14 == &no_more_args) goto done; args[n++] = &a14; + if (&a15 == &no_more_args) goto done; args[n++] = &a15; +done: + + size_t consumed; + int vec[kVecSize] = {}; + if (pattern.DoMatchImpl(*input, ANCHOR_START, &consumed, + args, n, vec, kVecSize)) { + input->remove_prefix(consumed); + return true; + } else { + return false; + } +} + +bool PCRE::FindAndConsumeFunctor::operator ()(StringPiece* input, + const PCRE& pattern, + const Arg& a0, + const Arg& a1, + const Arg& a2, + const Arg& a3, + const Arg& a4, + const Arg& a5, + const Arg& a6, + const Arg& a7, + const Arg& a8, + const Arg& a9, + const Arg& a10, + const Arg& a11, + const Arg& a12, + const Arg& a13, + const Arg& a14, + const Arg& a15) const { + const Arg* args[kMaxArgs]; + int n = 0; + if (&a0 == &no_more_args) goto done; args[n++] = &a0; + if (&a1 == &no_more_args) goto done; args[n++] = &a1; + if (&a2 == &no_more_args) goto done; args[n++] = &a2; + if (&a3 == &no_more_args) goto done; args[n++] = &a3; + if (&a4 == &no_more_args) goto done; args[n++] = &a4; + if (&a5 == &no_more_args) goto done; args[n++] = &a5; + if (&a6 == &no_more_args) goto done; args[n++] = &a6; + if (&a7 == &no_more_args) goto done; args[n++] = &a7; + if (&a8 == &no_more_args) goto done; args[n++] = &a8; + if (&a9 == &no_more_args) goto done; args[n++] = &a9; + if (&a10 == &no_more_args) goto done; args[n++] = &a10; + if (&a11 == &no_more_args) goto done; args[n++] = &a11; + if (&a12 == &no_more_args) goto done; args[n++] = &a12; + if (&a13 == &no_more_args) goto done; args[n++] = &a13; + if (&a14 == &no_more_args) goto done; args[n++] = &a14; + if (&a15 == &no_more_args) goto done; args[n++] = &a15; +done: + + size_t consumed; + int vec[kVecSize] = {}; + if (pattern.DoMatchImpl(*input, UNANCHORED, &consumed, + args, n, vec, kVecSize)) { + input->remove_prefix(consumed); + return true; + } else { + return false; + } +} + +bool PCRE::Replace(std::string *str, + const PCRE& pattern, + const StringPiece& rewrite) { + int vec[kVecSize] = {}; + int matches = pattern.TryMatch(*str, 0, UNANCHORED, true, vec, kVecSize); + if (matches == 0) + return false; + + std::string s; + if (!pattern.Rewrite(&s, rewrite, *str, vec, matches)) + return false; + + assert(vec[0] >= 0); + assert(vec[1] >= 0); + str->replace(vec[0], vec[1] - vec[0], s); + return true; +} + +int PCRE::GlobalReplace(std::string *str, + const PCRE& pattern, + const StringPiece& rewrite) { + int count = 0; + int vec[kVecSize] = {}; + std::string out; + size_t start = 0; + bool last_match_was_empty_string = false; + + while (start <= str->size()) { + // If the previous match was for the empty string, we shouldn't + // just match again: we'll match in the same way and get an + // infinite loop. Instead, we do the match in a special way: + // anchored -- to force another try at the same position -- + // and with a flag saying that this time, ignore empty matches. + // If this special match returns, that means there's a non-empty + // match at this position as well, and we can continue. If not, + // we do what perl does, and just advance by one. + // Notice that perl prints '@@@' for this; + // perl -le '$_ = "aa"; s/b*|aa/@/g; print' + int matches; + if (last_match_was_empty_string) { + matches = pattern.TryMatch(*str, start, ANCHOR_START, false, + vec, kVecSize); + if (matches <= 0) { + if (start < str->size()) + out.push_back((*str)[start]); + start++; + last_match_was_empty_string = false; + continue; + } + } else { + matches = pattern.TryMatch(*str, start, UNANCHORED, true, + vec, kVecSize); + if (matches <= 0) + break; + } + size_t matchstart = vec[0], matchend = vec[1]; + assert(matchstart >= start); + assert(matchend >= matchstart); + + out.append(*str, start, matchstart - start); + pattern.Rewrite(&out, rewrite, *str, vec, matches); + start = matchend; + count++; + last_match_was_empty_string = (matchstart == matchend); + } + + if (count == 0) + return 0; + + if (start < str->size()) + out.append(*str, start, str->size() - start); + using std::swap; + swap(out, *str); + return count; +} + +bool PCRE::Extract(const StringPiece &text, + const PCRE& pattern, + const StringPiece &rewrite, + std::string *out) { + int vec[kVecSize] = {}; + int matches = pattern.TryMatch(text, 0, UNANCHORED, true, vec, kVecSize); + if (matches == 0) + return false; + out->clear(); + return pattern.Rewrite(out, rewrite, text, vec, matches); +} + +std::string PCRE::QuoteMeta(const StringPiece& unquoted) { + std::string result; + result.reserve(unquoted.size() << 1); + + // Escape any ascii character not in [A-Za-z_0-9]. + // + // Note that it's legal to escape a character even if it has no + // special meaning in a regular expression -- so this function does + // that. (This also makes it identical to the perl function of the + // same name except for the null-character special case; + // see `perldoc -f quotemeta`.) + for (size_t ii = 0; ii < unquoted.size(); ++ii) { + // Note that using 'isalnum' here raises the benchmark time from + // 32ns to 58ns: + if ((unquoted[ii] < 'a' || unquoted[ii] > 'z') && + (unquoted[ii] < 'A' || unquoted[ii] > 'Z') && + (unquoted[ii] < '0' || unquoted[ii] > '9') && + unquoted[ii] != '_' && + // If this is the part of a UTF8 or Latin1 character, we need + // to copy this byte without escaping. Experimentally this is + // what works correctly with the regexp library. + !(unquoted[ii] & 128)) { + if (unquoted[ii] == '\0') { // Special handling for null chars. + // Can't use "\\0" since the next character might be a digit. + result += "\\x00"; + continue; + } + result += '\\'; + } + result += unquoted[ii]; + } + + return result; +} + +/***** Actual matching and rewriting code *****/ + +bool PCRE::HitLimit() { + return hit_limit_ != 0; +} + +void PCRE::ClearHitLimit() { + hit_limit_ = 0; +} + +int PCRE::TryMatch(const StringPiece& text, + size_t startpos, + Anchor anchor, + bool empty_ok, + int *vec, + int vecsize) const { + pcre* re = (anchor == ANCHOR_BOTH) ? re_full_ : re_partial_; + if (re == NULL) { + PCREPORT(ERROR) << "Matching against invalid re: " << *error_; + return 0; + } + + int match_limit = match_limit_; + if (match_limit <= 0) { + match_limit = FLAGS_regexp_match_limit; + } + + int stack_limit = stack_limit_; + if (stack_limit <= 0) { + stack_limit = FLAGS_regexp_stack_limit; + } + + pcre_extra extra = { 0 }; + if (match_limit > 0) { + extra.flags |= PCRE_EXTRA_MATCH_LIMIT; + extra.match_limit = match_limit; + } + if (stack_limit > 0) { + extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; + extra.match_limit_recursion = stack_limit / kPCREFrameSize; + } + + int options = 0; + if (anchor != UNANCHORED) + options |= PCRE_ANCHORED; + if (!empty_ok) + options |= PCRE_NOTEMPTY; + + int rc = pcre_exec(re, // The regular expression object + &extra, + (text.data() == NULL) ? "" : text.data(), + static_cast(text.size()), + static_cast(startpos), + options, + vec, + vecsize); + + // Handle errors + if (rc == 0) { + // pcre_exec() returns 0 as a special case when the number of + // capturing subpatterns exceeds the size of the vector. + // When this happens, there is a match and the output vector + // is filled, but we miss out on the positions of the extra subpatterns. + rc = vecsize / 2; + } else if (rc < 0) { + switch (rc) { + case PCRE_ERROR_NOMATCH: + return 0; + case PCRE_ERROR_MATCHLIMIT: + // Writing to hit_limit is not safe if multiple threads + // are using the PCRE, but the flag is only intended + // for use by unit tests anyway, so we let it go. + hit_limit_ = true; + PCREPORT(WARNING) << "Exceeded match limit of " << match_limit + << " when matching '" << pattern_ << "'" + << " against text that is " << text.size() << " bytes."; + return 0; + case PCRE_ERROR_RECURSIONLIMIT: + // See comment about hit_limit above. + hit_limit_ = true; + PCREPORT(WARNING) << "Exceeded stack limit of " << stack_limit + << " when matching '" << pattern_ << "'" + << " against text that is " << text.size() << " bytes."; + return 0; + default: + // There are other return codes from pcre.h : + // PCRE_ERROR_NULL (-2) + // PCRE_ERROR_BADOPTION (-3) + // PCRE_ERROR_BADMAGIC (-4) + // PCRE_ERROR_UNKNOWN_NODE (-5) + // PCRE_ERROR_NOMEMORY (-6) + // PCRE_ERROR_NOSUBSTRING (-7) + // ... + PCREPORT(ERROR) << "Unexpected return code: " << rc + << " when matching '" << pattern_ << "'" + << ", re=" << re + << ", text=" << text + << ", vec=" << vec + << ", vecsize=" << vecsize; + return 0; + } + } + + return rc; +} + +bool PCRE::DoMatchImpl(const StringPiece& text, + Anchor anchor, + size_t* consumed, + const Arg* const* args, + int n, + int* vec, + int vecsize) const { + assert((1 + n) * 3 <= vecsize); // results + PCRE workspace + if (NumberOfCapturingGroups() < n) { + // RE has fewer capturing groups than number of Arg pointers passed in. + return false; + } + + int matches = TryMatch(text, 0, anchor, true, vec, vecsize); + assert(matches >= 0); // TryMatch never returns negatives + if (matches == 0) + return false; + + *consumed = vec[1]; + + if (n == 0 || args == NULL) { + // We are not interested in results + return true; + } + + // If we got here, we must have matched the whole pattern. + // We do not need (can not do) any more checks on the value of 'matches' here + // -- see the comment for TryMatch. + for (int i = 0; i < n; i++) { + const int start = vec[2*(i+1)]; + const int limit = vec[2*(i+1)+1]; + + // Avoid invoking undefined behavior when text.data() happens + // to be null and start happens to be -1, the latter being the + // case for an unmatched subexpression. Even if text.data() is + // not null, pointing one byte before was a longstanding bug. + const char* addr = NULL; + if (start != -1) { + addr = text.data() + start; + } + + if (!args[i]->Parse(addr, limit-start)) { + // TODO: Should we indicate what the error was? + return false; + } + } + + return true; +} + +bool PCRE::DoMatch(const StringPiece& text, + Anchor anchor, + size_t* consumed, + const Arg* const args[], + int n) const { + assert(n >= 0); + const int vecsize = (1 + n) * 3; // results + PCRE workspace + // (as for kVecSize) + int* vec = new int[vecsize]; + bool b = DoMatchImpl(text, anchor, consumed, args, n, vec, vecsize); + delete[] vec; + return b; +} + +bool PCRE::Rewrite(std::string *out, const StringPiece &rewrite, + const StringPiece &text, int *vec, int veclen) const { + int number_of_capturing_groups = NumberOfCapturingGroups(); + for (const char *s = rewrite.data(), *end = s + rewrite.size(); + s < end; s++) { + int c = *s; + if (c == '\\') { + c = *++s; + if (isdigit(c)) { + int n = (c - '0'); + if (n >= veclen) { + if (n <= number_of_capturing_groups) { + // unmatched optional capturing group. treat + // its value as empty string; i.e., nothing to append. + } else { + PCREPORT(ERROR) << "requested group " << n + << " in regexp " << rewrite.data(); + return false; + } + } + int start = vec[2 * n]; + if (start >= 0) + out->append(text.data() + start, vec[2 * n + 1] - start); + } else if (c == '\\') { + out->push_back('\\'); + } else { + PCREPORT(ERROR) << "invalid rewrite pattern: " << rewrite.data(); + return false; + } + } else { + out->push_back(c); + } + } + return true; +} + +bool PCRE::CheckRewriteString(const StringPiece& rewrite, + std::string* error) const { + int max_token = -1; + for (const char *s = rewrite.data(), *end = s + rewrite.size(); + s < end; s++) { + int c = *s; + if (c != '\\') { + continue; + } + if (++s == end) { + *error = "Rewrite schema error: '\\' not allowed at end."; + return false; + } + c = *s; + if (c == '\\') { + continue; + } + if (!isdigit(c)) { + *error = "Rewrite schema error: " + "'\\' must be followed by a digit or '\\'."; + return false; + } + int n = (c - '0'); + if (max_token < n) { + max_token = n; + } + } + + if (max_token > NumberOfCapturingGroups()) { + *error = StringPrintf( + "Rewrite schema requests %d matches, but the regexp only has %d " + "parenthesized subexpressions.", + max_token, NumberOfCapturingGroups()); + return false; + } + return true; +} + + +// Return the number of capturing subpatterns, or -1 if the +// regexp wasn't valid on construction. +int PCRE::NumberOfCapturingGroups() const { + if (re_partial_ == NULL) return -1; + + int result; + int rc = pcre_fullinfo(re_partial_, // The regular expression object + NULL, // We did not study the pattern + PCRE_INFO_CAPTURECOUNT, + &result); + if (rc != 0) { + PCREPORT(ERROR) << "Unexpected return code: " << rc; + return -1; + } + return result; +} + + +/***** Parsers for various types *****/ + +bool PCRE::Arg::parse_null(const char* str, size_t n, void* dest) { + // We fail if somebody asked us to store into a non-NULL void* pointer + return (dest == NULL); +} + +bool PCRE::Arg::parse_string(const char* str, size_t n, void* dest) { + if (dest == NULL) return true; + reinterpret_cast(dest)->assign(str, n); + return true; +} + +bool PCRE::Arg::parse_stringpiece(const char* str, size_t n, void* dest) { + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = StringPiece(str, n); + return true; +} + +bool PCRE::Arg::parse_char(const char* str, size_t n, void* dest) { + if (n != 1) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = str[0]; + return true; +} + +bool PCRE::Arg::parse_schar(const char* str, size_t n, void* dest) { + if (n != 1) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = str[0]; + return true; +} + +bool PCRE::Arg::parse_uchar(const char* str, size_t n, void* dest) { + if (n != 1) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = str[0]; + return true; +} + +// Largest number spec that we are willing to parse +static const int kMaxNumberLength = 32; + +// PCREQUIPCRES "buf" must have length at least kMaxNumberLength+1 +// PCREQUIPCRES "n > 0" +// Copies "str" into "buf" and null-terminates if necessary. +// Returns one of: +// a. "str" if no termination is needed +// b. "buf" if the string was copied and null-terminated +// c. "" if the input was invalid and has no hope of being parsed +static const char* TerminateNumber(char* buf, const char* str, size_t n) { + if ((n > 0) && isspace(*str)) { + // We are less forgiving than the strtoxxx() routines and do not + // allow leading spaces. + return ""; + } + + // See if the character right after the input text may potentially + // look like a digit. + if (isdigit(str[n]) || + ((str[n] >= 'a') && (str[n] <= 'f')) || + ((str[n] >= 'A') && (str[n] <= 'F'))) { + if (n > kMaxNumberLength) return ""; // Input too big to be a valid number + memcpy(buf, str, n); + buf[n] = '\0'; + return buf; + } else { + // We can parse right out of the supplied string, so return it. + return str; + } +} + +bool PCRE::Arg::parse_long_radix(const char* str, + size_t n, + void* dest, + int radix) { + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, str, n); + char* end; + errno = 0; + long r = strtol(str, &end, radix); + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +bool PCRE::Arg::parse_ulong_radix(const char* str, + size_t n, + void* dest, + int radix) { + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, str, n); + if (str[0] == '-') { + // strtoul() will silently accept negative numbers and parse + // them. This module is more strict and treats them as errors. + return false; + } + + char* end; + errno = 0; + unsigned long r = strtoul(str, &end, radix); + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +bool PCRE::Arg::parse_short_radix(const char* str, + size_t n, + void* dest, + int radix) { + long r; + if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse + if ((short)r != r) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = (short)r; + return true; +} + +bool PCRE::Arg::parse_ushort_radix(const char* str, + size_t n, + void* dest, + int radix) { + unsigned long r; + if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse + if ((unsigned short)r != r) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = (unsigned short)r; + return true; +} + +bool PCRE::Arg::parse_int_radix(const char* str, + size_t n, + void* dest, + int radix) { + long r; + if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse + if ((int)r != r) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = (int)r; + return true; +} + +bool PCRE::Arg::parse_uint_radix(const char* str, + size_t n, + void* dest, + int radix) { + unsigned long r; + if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse + if ((unsigned int)r != r) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = (unsigned int)r; + return true; +} + +bool PCRE::Arg::parse_longlong_radix(const char* str, + size_t n, + void* dest, + int radix) { + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, str, n); + char* end; + errno = 0; + long long r = strtoll(str, &end, radix); + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +bool PCRE::Arg::parse_ulonglong_radix(const char* str, + size_t n, + void* dest, + int radix) { + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, str, n); + if (str[0] == '-') { + // strtoull() will silently accept negative numbers and parse + // them. This module is more strict and treats them as errors. + return false; + } + char* end; + errno = 0; + unsigned long long r = strtoull(str, &end, radix); + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +static bool parse_double_float(const char* str, size_t n, bool isfloat, + void* dest) { + if (n == 0) return false; + static const int kMaxLength = 200; + char buf[kMaxLength]; + if (n >= kMaxLength) return false; + memcpy(buf, str, n); + buf[n] = '\0'; + char* end; + errno = 0; + double r; + if (isfloat) { + r = strtof(buf, &end); + } else { + r = strtod(buf, &end); + } + if (end != buf + n) { +#ifdef _WIN32 + // Microsoft's strtod() doesn't handle inf and nan, so we have to + // handle it explicitly. Speed is not important here because this + // code is only called in unit tests. + bool pos = true; + const char* i = buf; + if ('-' == *i) { + pos = false; + ++i; + } else if ('+' == *i) { + ++i; + } + if (0 == _stricmp(i, "inf") || 0 == _stricmp(i, "infinity")) { + r = std::numeric_limits::infinity(); + if (!pos) + r = -r; + } else if (0 == _stricmp(i, "nan")) { + r = std::numeric_limits::quiet_NaN(); + } else { + return false; + } +#else + return false; // Leftover junk +#endif + } + if (errno) return false; + if (dest == NULL) return true; + if (isfloat) { + *(reinterpret_cast(dest)) = (float)r; + } else { + *(reinterpret_cast(dest)) = r; + } + return true; +} + +bool PCRE::Arg::parse_double(const char* str, size_t n, void* dest) { + return parse_double_float(str, n, false, dest); +} + +bool PCRE::Arg::parse_float(const char* str, size_t n, void* dest) { + return parse_double_float(str, n, true, dest); +} + +#define DEFINE_INTEGER_PARSER(name) \ + bool PCRE::Arg::parse_##name(const char* str, size_t n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 10); \ + } \ + bool PCRE::Arg::parse_##name##_hex(const char* str, size_t n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 16); \ + } \ + bool PCRE::Arg::parse_##name##_octal(const char* str, size_t n, \ + void* dest) { \ + return parse_##name##_radix(str, n, dest, 8); \ + } \ + bool PCRE::Arg::parse_##name##_cradix(const char* str, size_t n, \ + void* dest) { \ + return parse_##name##_radix(str, n, dest, 0); \ + } + +DEFINE_INTEGER_PARSER(short); +DEFINE_INTEGER_PARSER(ushort); +DEFINE_INTEGER_PARSER(int); +DEFINE_INTEGER_PARSER(uint); +DEFINE_INTEGER_PARSER(long); +DEFINE_INTEGER_PARSER(ulong); +DEFINE_INTEGER_PARSER(longlong); +DEFINE_INTEGER_PARSER(ulonglong); + +#undef DEFINE_INTEGER_PARSER + +} // namespace re2 diff --git a/extern/re2/util/pcre.h b/extern/re2/util/pcre.h new file mode 100644 index 0000000000..644dce68c2 --- /dev/null +++ b/extern/re2/util/pcre.h @@ -0,0 +1,681 @@ +// Copyright 2003-2010 Google Inc. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef UTIL_PCRE_H_ +#define UTIL_PCRE_H_ + +// This is a variant of PCRE's pcrecpp.h, originally written at Google. +// The main changes are the addition of the HitLimit method and +// compilation as PCRE in namespace re2. + +// C++ interface to the pcre regular-expression library. PCRE supports +// Perl-style regular expressions (with extensions like \d, \w, \s, +// ...). +// +// ----------------------------------------------------------------------- +// REGEXP SYNTAX: +// +// This module uses the pcre library and hence supports its syntax +// for regular expressions: +// +// http://www.google.com/search?q=pcre +// +// The syntax is pretty similar to Perl's. For those not familiar +// with Perl's regular expressions, here are some examples of the most +// commonly used extensions: +// +// "hello (\\w+) world" -- \w matches a "word" character +// "version (\\d+)" -- \d matches a digit +// "hello\\s+world" -- \s matches any whitespace character +// "\\b(\\w+)\\b" -- \b matches empty string at a word boundary +// "(?i)hello" -- (?i) turns on case-insensitive matching +// "/\\*(.*?)\\*/" -- .*? matches . minimum no. of times possible +// +// ----------------------------------------------------------------------- +// MATCHING INTERFACE: +// +// The "FullMatch" operation checks that supplied text matches a +// supplied pattern exactly. +// +// Example: successful match +// CHECK(PCRE::FullMatch("hello", "h.*o")); +// +// Example: unsuccessful match (requires full match): +// CHECK(!PCRE::FullMatch("hello", "e")); +// +// ----------------------------------------------------------------------- +// UTF-8 AND THE MATCHING INTERFACE: +// +// By default, pattern and text are plain text, one byte per character. +// The UTF8 flag, passed to the constructor, causes both pattern +// and string to be treated as UTF-8 text, still a byte stream but +// potentially multiple bytes per character. In practice, the text +// is likelier to be UTF-8 than the pattern, but the match returned +// may depend on the UTF8 flag, so always use it when matching +// UTF8 text. E.g., "." will match one byte normally but with UTF8 +// set may match up to three bytes of a multi-byte character. +// +// Example: +// PCRE re(utf8_pattern, PCRE::UTF8); +// CHECK(PCRE::FullMatch(utf8_string, re)); +// +// ----------------------------------------------------------------------- +// MATCHING WITH SUBSTRING EXTRACTION: +// +// You can supply extra pointer arguments to extract matched substrings. +// +// Example: extracts "ruby" into "s" and 1234 into "i" +// int i; +// std::string s; +// CHECK(PCRE::FullMatch("ruby:1234", "(\\w+):(\\d+)", &s, &i)); +// +// Example: fails because string cannot be stored in integer +// CHECK(!PCRE::FullMatch("ruby", "(.*)", &i)); +// +// Example: fails because there aren't enough sub-patterns: +// CHECK(!PCRE::FullMatch("ruby:1234", "\\w+:\\d+", &s)); +// +// Example: does not try to extract any extra sub-patterns +// CHECK(PCRE::FullMatch("ruby:1234", "(\\w+):(\\d+)", &s)); +// +// Example: does not try to extract into NULL +// CHECK(PCRE::FullMatch("ruby:1234", "(\\w+):(\\d+)", NULL, &i)); +// +// Example: integer overflow causes failure +// CHECK(!PCRE::FullMatch("ruby:1234567891234", "\\w+:(\\d+)", &i)); +// +// ----------------------------------------------------------------------- +// PARTIAL MATCHES +// +// You can use the "PartialMatch" operation when you want the pattern +// to match any substring of the text. +// +// Example: simple search for a string: +// CHECK(PCRE::PartialMatch("hello", "ell")); +// +// Example: find first number in a string +// int number; +// CHECK(PCRE::PartialMatch("x*100 + 20", "(\\d+)", &number)); +// CHECK_EQ(number, 100); +// +// ----------------------------------------------------------------------- +// PPCRE-COMPILED PCREGULAR EXPPCRESSIONS +// +// PCRE makes it easy to use any string as a regular expression, without +// requiring a separate compilation step. +// +// If speed is of the essence, you can create a pre-compiled "PCRE" +// object from the pattern and use it multiple times. If you do so, +// you can typically parse text faster than with sscanf. +// +// Example: precompile pattern for faster matching: +// PCRE pattern("h.*o"); +// while (ReadLine(&str)) { +// if (PCRE::FullMatch(str, pattern)) ...; +// } +// +// ----------------------------------------------------------------------- +// SCANNING TEXT INCPCREMENTALLY +// +// The "Consume" operation may be useful if you want to repeatedly +// match regular expressions at the front of a string and skip over +// them as they match. This requires use of the "StringPiece" type, +// which represents a sub-range of a real string. +// +// Example: read lines of the form "var = value" from a string. +// std::string contents = ...; // Fill string somehow +// StringPiece input(contents); // Wrap a StringPiece around it +// +// std::string var; +// int value; +// while (PCRE::Consume(&input, "(\\w+) = (\\d+)\n", &var, &value)) { +// ...; +// } +// +// Each successful call to "Consume" will set "var/value", and also +// advance "input" so it points past the matched text. Note that if the +// regular expression matches an empty string, input will advance +// by 0 bytes. If the regular expression being used might match +// an empty string, the loop body must check for this case and either +// advance the string or break out of the loop. +// +// The "FindAndConsume" operation is similar to "Consume" but does not +// anchor your match at the beginning of the string. For example, you +// could extract all words from a string by repeatedly calling +// PCRE::FindAndConsume(&input, "(\\w+)", &word) +// +// ----------------------------------------------------------------------- +// PARSING HEX/OCTAL/C-RADIX NUMBERS +// +// By default, if you pass a pointer to a numeric value, the +// corresponding text is interpreted as a base-10 number. You can +// instead wrap the pointer with a call to one of the operators Hex(), +// Octal(), or CRadix() to interpret the text in another base. The +// CRadix operator interprets C-style "0" (base-8) and "0x" (base-16) +// prefixes, but defaults to base-10. +// +// Example: +// int a, b, c, d; +// CHECK(PCRE::FullMatch("100 40 0100 0x40", "(.*) (.*) (.*) (.*)", +// Octal(&a), Hex(&b), CRadix(&c), CRadix(&d)); +// will leave 64 in a, b, c, and d. + +#include "util/util.h" +#include "re2/stringpiece.h" + +#ifdef USEPCRE +#include +namespace re2 { +const bool UsingPCRE = true; +} // namespace re2 +#else +struct pcre; // opaque +namespace re2 { +const bool UsingPCRE = false; +} // namespace re2 +#endif + +namespace re2 { + +class PCRE_Options; + +// Interface for regular expression matching. Also corresponds to a +// pre-compiled regular expression. An "PCRE" object is safe for +// concurrent use by multiple threads. +class PCRE { + public: + // We convert user-passed pointers into special Arg objects + class Arg; + + // Marks end of arg list. + // ONLY USE IN OPTIONAL ARG DEFAULTS. + // DO NOT PASS EXPLICITLY. + static Arg no_more_args; + + // Options are same value as those in pcre. We provide them here + // to avoid users needing to include pcre.h and also to isolate + // users from pcre should we change the underlying library. + // Only those needed by Google programs are exposed here to + // avoid collision with options employed internally by regexp.cc + // Note that some options have equivalents that can be specified in + // the regexp itself. For example, prefixing your regexp with + // "(?s)" has the same effect as the PCRE_DOTALL option. + enum Option { + None = 0x0000, + UTF8 = 0x0800, // == PCRE_UTF8 + EnabledCompileOptions = UTF8, + EnabledExecOptions = 0x0000, // TODO: use to replace anchor flag + }; + + // We provide implicit conversions from strings so that users can + // pass in a string or a "const char*" wherever an "PCRE" is expected. + PCRE(const char* pattern); + PCRE(const char* pattern, Option option); + PCRE(const std::string& pattern); + PCRE(const std::string& pattern, Option option); + PCRE(const char *pattern, const PCRE_Options& re_option); + PCRE(const std::string& pattern, const PCRE_Options& re_option); + + ~PCRE(); + + // The string specification for this PCRE. E.g. + // PCRE re("ab*c?d+"); + // re.pattern(); // "ab*c?d+" + const std::string& pattern() const { return pattern_; } + + // If PCRE could not be created properly, returns an error string. + // Else returns the empty string. + const std::string& error() const { return *error_; } + + // Whether the PCRE has hit a match limit during execution. + // Not thread safe. Intended only for testing. + // If hitting match limits is a problem, + // you should be using PCRE2 (re2/re2.h) + // instead of checking this flag. + bool HitLimit(); + void ClearHitLimit(); + + /***** The useful part: the matching interface *****/ + + // Matches "text" against "pattern". If pointer arguments are + // supplied, copies matched sub-patterns into them. + // + // You can pass in a "const char*" or a "std::string" for "text". + // You can pass in a "const char*" or a "std::string" or a "PCRE" for "pattern". + // + // The provided pointer arguments can be pointers to any scalar numeric + // type, or one of: + // std::string (matched piece is copied to string) + // StringPiece (StringPiece is mutated to point to matched piece) + // T (where "bool T::ParseFrom(const char*, size_t)" exists) + // (void*)NULL (the corresponding matched sub-pattern is not copied) + // + // Returns true iff all of the following conditions are satisfied: + // a. "text" matches "pattern" exactly + // b. The number of matched sub-patterns is >= number of supplied pointers + // c. The "i"th argument has a suitable type for holding the + // string captured as the "i"th sub-pattern. If you pass in + // NULL for the "i"th argument, or pass fewer arguments than + // number of sub-patterns, "i"th captured sub-pattern is + // ignored. + // + // CAVEAT: An optional sub-pattern that does not exist in the + // matched string is assigned the empty string. Therefore, the + // following will return false (because the empty string is not a + // valid number): + // int number; + // PCRE::FullMatch("abc", "[a-z]+(\\d+)?", &number); + struct FullMatchFunctor { + bool operator ()(const StringPiece& text, const PCRE& re, // 3..16 args + const Arg& ptr1 = no_more_args, + const Arg& ptr2 = no_more_args, + const Arg& ptr3 = no_more_args, + const Arg& ptr4 = no_more_args, + const Arg& ptr5 = no_more_args, + const Arg& ptr6 = no_more_args, + const Arg& ptr7 = no_more_args, + const Arg& ptr8 = no_more_args, + const Arg& ptr9 = no_more_args, + const Arg& ptr10 = no_more_args, + const Arg& ptr11 = no_more_args, + const Arg& ptr12 = no_more_args, + const Arg& ptr13 = no_more_args, + const Arg& ptr14 = no_more_args, + const Arg& ptr15 = no_more_args, + const Arg& ptr16 = no_more_args) const; + }; + + static const FullMatchFunctor FullMatch; + + // Exactly like FullMatch(), except that "pattern" is allowed to match + // a substring of "text". + struct PartialMatchFunctor { + bool operator ()(const StringPiece& text, const PCRE& re, // 3..16 args + const Arg& ptr1 = no_more_args, + const Arg& ptr2 = no_more_args, + const Arg& ptr3 = no_more_args, + const Arg& ptr4 = no_more_args, + const Arg& ptr5 = no_more_args, + const Arg& ptr6 = no_more_args, + const Arg& ptr7 = no_more_args, + const Arg& ptr8 = no_more_args, + const Arg& ptr9 = no_more_args, + const Arg& ptr10 = no_more_args, + const Arg& ptr11 = no_more_args, + const Arg& ptr12 = no_more_args, + const Arg& ptr13 = no_more_args, + const Arg& ptr14 = no_more_args, + const Arg& ptr15 = no_more_args, + const Arg& ptr16 = no_more_args) const; + }; + + static const PartialMatchFunctor PartialMatch; + + // Like FullMatch() and PartialMatch(), except that pattern has to + // match a prefix of "text", and "input" is advanced past the matched + // text. Note: "input" is modified iff this routine returns true. + struct ConsumeFunctor { + bool operator ()(StringPiece* input, const PCRE& pattern, // 3..16 args + const Arg& ptr1 = no_more_args, + const Arg& ptr2 = no_more_args, + const Arg& ptr3 = no_more_args, + const Arg& ptr4 = no_more_args, + const Arg& ptr5 = no_more_args, + const Arg& ptr6 = no_more_args, + const Arg& ptr7 = no_more_args, + const Arg& ptr8 = no_more_args, + const Arg& ptr9 = no_more_args, + const Arg& ptr10 = no_more_args, + const Arg& ptr11 = no_more_args, + const Arg& ptr12 = no_more_args, + const Arg& ptr13 = no_more_args, + const Arg& ptr14 = no_more_args, + const Arg& ptr15 = no_more_args, + const Arg& ptr16 = no_more_args) const; + }; + + static const ConsumeFunctor Consume; + + // Like Consume(..), but does not anchor the match at the beginning of the + // string. That is, "pattern" need not start its match at the beginning of + // "input". For example, "FindAndConsume(s, "(\\w+)", &word)" finds the next + // word in "s" and stores it in "word". + struct FindAndConsumeFunctor { + bool operator ()(StringPiece* input, const PCRE& pattern, + const Arg& ptr1 = no_more_args, + const Arg& ptr2 = no_more_args, + const Arg& ptr3 = no_more_args, + const Arg& ptr4 = no_more_args, + const Arg& ptr5 = no_more_args, + const Arg& ptr6 = no_more_args, + const Arg& ptr7 = no_more_args, + const Arg& ptr8 = no_more_args, + const Arg& ptr9 = no_more_args, + const Arg& ptr10 = no_more_args, + const Arg& ptr11 = no_more_args, + const Arg& ptr12 = no_more_args, + const Arg& ptr13 = no_more_args, + const Arg& ptr14 = no_more_args, + const Arg& ptr15 = no_more_args, + const Arg& ptr16 = no_more_args) const; + }; + + static const FindAndConsumeFunctor FindAndConsume; + + // Replace the first match of "pattern" in "str" with "rewrite". + // Within "rewrite", backslash-escaped digits (\1 to \9) can be + // used to insert text matching corresponding parenthesized group + // from the pattern. \0 in "rewrite" refers to the entire matching + // text. E.g., + // + // std::string s = "yabba dabba doo"; + // CHECK(PCRE::Replace(&s, "b+", "d")); + // + // will leave "s" containing "yada dabba doo" + // + // Returns true if the pattern matches and a replacement occurs, + // false otherwise. + static bool Replace(std::string *str, + const PCRE& pattern, + const StringPiece& rewrite); + + // Like Replace(), except replaces all occurrences of the pattern in + // the string with the rewrite. Replacements are not subject to + // re-matching. E.g., + // + // std::string s = "yabba dabba doo"; + // CHECK(PCRE::GlobalReplace(&s, "b+", "d")); + // + // will leave "s" containing "yada dada doo" + // + // Returns the number of replacements made. + static int GlobalReplace(std::string *str, + const PCRE& pattern, + const StringPiece& rewrite); + + // Like Replace, except that if the pattern matches, "rewrite" + // is copied into "out" with substitutions. The non-matching + // portions of "text" are ignored. + // + // Returns true iff a match occurred and the extraction happened + // successfully; if no match occurs, the string is left unaffected. + static bool Extract(const StringPiece &text, + const PCRE& pattern, + const StringPiece &rewrite, + std::string *out); + + // Check that the given @p rewrite string is suitable for use with + // this PCRE. It checks that: + // * The PCRE has enough parenthesized subexpressions to satisfy all + // of the \N tokens in @p rewrite, and + // * The @p rewrite string doesn't have any syntax errors + // ('\' followed by anything besides [0-9] and '\'). + // Making this test will guarantee that "replace" and "extract" + // operations won't LOG(ERROR) or fail because of a bad rewrite + // string. + // @param rewrite The proposed rewrite string. + // @param error An error message is recorded here, iff we return false. + // Otherwise, it is unchanged. + // @return true, iff @p rewrite is suitable for use with the PCRE. + bool CheckRewriteString(const StringPiece& rewrite, + std::string* error) const; + + // Returns a copy of 'unquoted' with all potentially meaningful + // regexp characters backslash-escaped. The returned string, used + // as a regular expression, will exactly match the original string. + // For example, + // 1.5-2.0? + // becomes: + // 1\.5\-2\.0\? + static std::string QuoteMeta(const StringPiece& unquoted); + + /***** Generic matching interface (not so nice to use) *****/ + + // Type of match (TODO: Should be restructured as an Option) + enum Anchor { + UNANCHORED, // No anchoring + ANCHOR_START, // Anchor at start only + ANCHOR_BOTH, // Anchor at start and end + }; + + // General matching routine. Stores the length of the match in + // "*consumed" if successful. + bool DoMatch(const StringPiece& text, + Anchor anchor, + size_t* consumed, + const Arg* const* args, int n) const; + + // Return the number of capturing subpatterns, or -1 if the + // regexp wasn't valid on construction. + int NumberOfCapturingGroups() const; + + private: + void Init(const char* pattern, Option option, int match_limit, + int stack_limit, bool report_errors); + + // Match against "text", filling in "vec" (up to "vecsize" * 2/3) with + // pairs of integers for the beginning and end positions of matched + // text. The first pair corresponds to the entire matched text; + // subsequent pairs correspond, in order, to parentheses-captured + // matches. Returns the number of pairs (one more than the number of + // the last subpattern with a match) if matching was successful + // and zero if the match failed. + // I.e. for PCRE("(foo)|(bar)|(baz)") it will return 2, 3, and 4 when matching + // against "foo", "bar", and "baz" respectively. + // When matching PCRE("(foo)|hello") against "hello", it will return 1. + // But the values for all subpattern are filled in into "vec". + int TryMatch(const StringPiece& text, + size_t startpos, + Anchor anchor, + bool empty_ok, + int *vec, + int vecsize) const; + + // Append the "rewrite" string, with backslash subsitutions from "text" + // and "vec", to string "out". + bool Rewrite(std::string *out, + const StringPiece &rewrite, + const StringPiece &text, + int *vec, + int veclen) const; + + // internal implementation for DoMatch + bool DoMatchImpl(const StringPiece& text, + Anchor anchor, + size_t* consumed, + const Arg* const args[], + int n, + int* vec, + int vecsize) const; + + // Compile the regexp for the specified anchoring mode + pcre* Compile(Anchor anchor); + + std::string pattern_; + Option options_; + pcre* re_full_; // For full matches + pcre* re_partial_; // For partial matches + const std::string* error_; // Error indicator (or empty string) + bool report_errors_; // Silences error logging if false + int match_limit_; // Limit on execution resources + int stack_limit_; // Limit on stack resources (bytes) + mutable int32_t hit_limit_; // Hit limit during execution (bool) + + PCRE(const PCRE&) = delete; + PCRE& operator=(const PCRE&) = delete; +}; + +// PCRE_Options allow you to set the PCRE::Options, plus any pcre +// "extra" options. The only extras are match_limit, which limits +// the CPU time of a match, and stack_limit, which limits the +// stack usage. Setting a limit to <= 0 lets PCRE pick a sensible default +// that should not cause too many problems in production code. +// If PCRE hits a limit during a match, it may return a false negative, +// but (hopefully) it won't crash. +// +// NOTE: If you are handling regular expressions specified by +// (external or internal) users, rather than hard-coded ones, +// you should be using PCRE2, which uses an alternate implementation +// that avoids these issues. See http://go/re2quick. +class PCRE_Options { + public: + // constructor + PCRE_Options() : option_(PCRE::None), match_limit_(0), stack_limit_(0), report_errors_(true) {} + // accessors + PCRE::Option option() const { return option_; } + void set_option(PCRE::Option option) { + option_ = option; + } + int match_limit() const { return match_limit_; } + void set_match_limit(int match_limit) { + match_limit_ = match_limit; + } + int stack_limit() const { return stack_limit_; } + void set_stack_limit(int stack_limit) { + stack_limit_ = stack_limit; + } + + // If the regular expression is malformed, an error message will be printed + // iff report_errors() is true. Default: true. + bool report_errors() const { return report_errors_; } + void set_report_errors(bool report_errors) { + report_errors_ = report_errors; + } + private: + PCRE::Option option_; + int match_limit_; + int stack_limit_; + bool report_errors_; +}; + + +/***** Implementation details *****/ + +// Hex/Octal/Binary? + +// Special class for parsing into objects that define a ParseFrom() method +template +class _PCRE_MatchObject { + public: + static inline bool Parse(const char* str, size_t n, void* dest) { + if (dest == NULL) return true; + T* object = reinterpret_cast(dest); + return object->ParseFrom(str, n); + } +}; + +class PCRE::Arg { + public: + // Empty constructor so we can declare arrays of PCRE::Arg + Arg(); + + // Constructor specially designed for NULL arguments + Arg(void*); + + typedef bool (*Parser)(const char* str, size_t n, void* dest); + +// Type-specific parsers +#define MAKE_PARSER(type, name) \ + Arg(type* p) : arg_(p), parser_(name) {} \ + Arg(type* p, Parser parser) : arg_(p), parser_(parser) {} + + MAKE_PARSER(char, parse_char); + MAKE_PARSER(signed char, parse_schar); + MAKE_PARSER(unsigned char, parse_uchar); + MAKE_PARSER(float, parse_float); + MAKE_PARSER(double, parse_double); + MAKE_PARSER(std::string, parse_string); + MAKE_PARSER(StringPiece, parse_stringpiece); + + MAKE_PARSER(short, parse_short); + MAKE_PARSER(unsigned short, parse_ushort); + MAKE_PARSER(int, parse_int); + MAKE_PARSER(unsigned int, parse_uint); + MAKE_PARSER(long, parse_long); + MAKE_PARSER(unsigned long, parse_ulong); + MAKE_PARSER(long long, parse_longlong); + MAKE_PARSER(unsigned long long, parse_ulonglong); + +#undef MAKE_PARSER + + // Generic constructor + template Arg(T*, Parser parser); + // Generic constructor template + template Arg(T* p) + : arg_(p), parser_(_PCRE_MatchObject::Parse) { + } + + // Parse the data + bool Parse(const char* str, size_t n) const; + + private: + void* arg_; + Parser parser_; + + static bool parse_null (const char* str, size_t n, void* dest); + static bool parse_char (const char* str, size_t n, void* dest); + static bool parse_schar (const char* str, size_t n, void* dest); + static bool parse_uchar (const char* str, size_t n, void* dest); + static bool parse_float (const char* str, size_t n, void* dest); + static bool parse_double (const char* str, size_t n, void* dest); + static bool parse_string (const char* str, size_t n, void* dest); + static bool parse_stringpiece (const char* str, size_t n, void* dest); + +#define DECLARE_INTEGER_PARSER(name) \ + private: \ + static bool parse_##name(const char* str, size_t n, void* dest); \ + static bool parse_##name##_radix(const char* str, size_t n, void* dest, \ + int radix); \ + \ + public: \ + static bool parse_##name##_hex(const char* str, size_t n, void* dest); \ + static bool parse_##name##_octal(const char* str, size_t n, void* dest); \ + static bool parse_##name##_cradix(const char* str, size_t n, void* dest) + + DECLARE_INTEGER_PARSER(short); + DECLARE_INTEGER_PARSER(ushort); + DECLARE_INTEGER_PARSER(int); + DECLARE_INTEGER_PARSER(uint); + DECLARE_INTEGER_PARSER(long); + DECLARE_INTEGER_PARSER(ulong); + DECLARE_INTEGER_PARSER(longlong); + DECLARE_INTEGER_PARSER(ulonglong); + +#undef DECLARE_INTEGER_PARSER + +}; + +inline PCRE::Arg::Arg() : arg_(NULL), parser_(parse_null) { } +inline PCRE::Arg::Arg(void* p) : arg_(p), parser_(parse_null) { } + +inline bool PCRE::Arg::Parse(const char* str, size_t n) const { + return (*parser_)(str, n, arg_); +} + +// This part of the parser, appropriate only for ints, deals with bases +#define MAKE_INTEGER_PARSER(type, name) \ + inline PCRE::Arg Hex(type* ptr) { \ + return PCRE::Arg(ptr, PCRE::Arg::parse_##name##_hex); \ + } \ + inline PCRE::Arg Octal(type* ptr) { \ + return PCRE::Arg(ptr, PCRE::Arg::parse_##name##_octal); \ + } \ + inline PCRE::Arg CRadix(type* ptr) { \ + return PCRE::Arg(ptr, PCRE::Arg::parse_##name##_cradix); \ + } + +MAKE_INTEGER_PARSER(short, short); +MAKE_INTEGER_PARSER(unsigned short, ushort); +MAKE_INTEGER_PARSER(int, int); +MAKE_INTEGER_PARSER(unsigned int, uint); +MAKE_INTEGER_PARSER(long, long); +MAKE_INTEGER_PARSER(unsigned long, ulong); +MAKE_INTEGER_PARSER(long long, longlong); +MAKE_INTEGER_PARSER(unsigned long long, ulonglong); + +#undef MAKE_INTEGER_PARSER + +} // namespace re2 + +#endif // UTIL_PCRE_H_ diff --git a/extern/re2/util/pod_array.h b/extern/re2/util/pod_array.h new file mode 100644 index 0000000000..eaf492d0ed --- /dev/null +++ b/extern/re2/util/pod_array.h @@ -0,0 +1,55 @@ +// Copyright 2018 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef UTIL_POD_ARRAY_H_ +#define UTIL_POD_ARRAY_H_ + +#include +#include + +namespace re2 { + +template +class PODArray { + public: + static_assert(std::is_pod::value, + "T must be POD"); + + PODArray() + : ptr_() {} + explicit PODArray(int len) + : ptr_(std::allocator().allocate(len), Deleter(len)) {} + + T* data() const { + return ptr_.get(); + } + + int size() const { + return ptr_.get_deleter().len_; + } + + T& operator[](int pos) const { + return ptr_[pos]; + } + + private: + struct Deleter { + Deleter() + : len_(0) {} + explicit Deleter(int len) + : len_(len) {} + + void operator()(T* ptr) const { + std::allocator().deallocate(ptr, len_); + } + + int len_; + }; + + std::unique_ptr ptr_; +}; + +} // namespace re2 + +#endif // UTIL_POD_ARRAY_H_ diff --git a/extern/re2/util/rune.cc b/extern/re2/util/rune.cc new file mode 100644 index 0000000000..4f625ea380 --- /dev/null +++ b/extern/re2/util/rune.cc @@ -0,0 +1,260 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson. + * Copyright (c) 2002 by Lucent Technologies. + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES MAKE ANY + * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY + * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include + +#include "util/utf.h" + +namespace re2 { + +enum +{ + Bit1 = 7, + Bitx = 6, + Bit2 = 5, + Bit3 = 4, + Bit4 = 3, + Bit5 = 2, + + T1 = ((1<<(Bit1+1))-1) ^ 0xFF, /* 0000 0000 */ + Tx = ((1<<(Bitx+1))-1) ^ 0xFF, /* 1000 0000 */ + T2 = ((1<<(Bit2+1))-1) ^ 0xFF, /* 1100 0000 */ + T3 = ((1<<(Bit3+1))-1) ^ 0xFF, /* 1110 0000 */ + T4 = ((1<<(Bit4+1))-1) ^ 0xFF, /* 1111 0000 */ + T5 = ((1<<(Bit5+1))-1) ^ 0xFF, /* 1111 1000 */ + + Rune1 = (1<<(Bit1+0*Bitx))-1, /* 0000 0000 0111 1111 */ + Rune2 = (1<<(Bit2+1*Bitx))-1, /* 0000 0111 1111 1111 */ + Rune3 = (1<<(Bit3+2*Bitx))-1, /* 1111 1111 1111 1111 */ + Rune4 = (1<<(Bit4+3*Bitx))-1, + /* 0001 1111 1111 1111 1111 1111 */ + + Maskx = (1< T1 + */ + c = *(unsigned char*)str; + if(c < Tx) { + *rune = c; + return 1; + } + + /* + * two character sequence + * 0080-07FF => T2 Tx + */ + c1 = *(unsigned char*)(str+1) ^ Tx; + if(c1 & Testx) + goto bad; + if(c < T3) { + if(c < T2) + goto bad; + l = ((c << Bitx) | c1) & Rune2; + if(l <= Rune1) + goto bad; + *rune = l; + return 2; + } + + /* + * three character sequence + * 0800-FFFF => T3 Tx Tx + */ + c2 = *(unsigned char*)(str+2) ^ Tx; + if(c2 & Testx) + goto bad; + if(c < T4) { + l = ((((c << Bitx) | c1) << Bitx) | c2) & Rune3; + if(l <= Rune2) + goto bad; + *rune = l; + return 3; + } + + /* + * four character sequence (21-bit value) + * 10000-1FFFFF => T4 Tx Tx Tx + */ + c3 = *(unsigned char*)(str+3) ^ Tx; + if (c3 & Testx) + goto bad; + if (c < T5) { + l = ((((((c << Bitx) | c1) << Bitx) | c2) << Bitx) | c3) & Rune4; + if (l <= Rune3) + goto bad; + *rune = l; + return 4; + } + + /* + * Support for 5-byte or longer UTF-8 would go here, but + * since we don't have that, we'll just fall through to bad. + */ + + /* + * bad decoding + */ +bad: + *rune = Bad; + return 1; +} + +int +runetochar(char *str, const Rune *rune) +{ + /* Runes are signed, so convert to unsigned for range check. */ + unsigned long c; + + /* + * one character sequence + * 00000-0007F => 00-7F + */ + c = *rune; + if(c <= Rune1) { + str[0] = static_cast(c); + return 1; + } + + /* + * two character sequence + * 0080-07FF => T2 Tx + */ + if(c <= Rune2) { + str[0] = T2 | static_cast(c >> 1*Bitx); + str[1] = Tx | (c & Maskx); + return 2; + } + + /* + * If the Rune is out of range, convert it to the error rune. + * Do this test here because the error rune encodes to three bytes. + * Doing it earlier would duplicate work, since an out of range + * Rune wouldn't have fit in one or two bytes. + */ + if (c > Runemax) + c = Runeerror; + + /* + * three character sequence + * 0800-FFFF => T3 Tx Tx + */ + if (c <= Rune3) { + str[0] = T3 | static_cast(c >> 2*Bitx); + str[1] = Tx | ((c >> 1*Bitx) & Maskx); + str[2] = Tx | (c & Maskx); + return 3; + } + + /* + * four character sequence (21-bit value) + * 10000-1FFFFF => T4 Tx Tx Tx + */ + str[0] = T4 | static_cast(c >> 3*Bitx); + str[1] = Tx | ((c >> 2*Bitx) & Maskx); + str[2] = Tx | ((c >> 1*Bitx) & Maskx); + str[3] = Tx | (c & Maskx); + return 4; +} + +int +runelen(Rune rune) +{ + char str[10]; + + return runetochar(str, &rune); +} + +int +fullrune(const char *str, int n) +{ + if (n > 0) { + int c = *(unsigned char*)str; + if (c < Tx) + return 1; + if (n > 1) { + if (c < T3) + return 1; + if (n > 2) { + if (c < T4 || n > 3) + return 1; + } + } + } + return 0; +} + + +int +utflen(const char *s) +{ + int c; + long n; + Rune rune; + + n = 0; + for(;;) { + c = *(unsigned char*)s; + if(c < Runeself) { + if(c == 0) + return n; + s++; + } else + s += chartorune(&rune, s); + n++; + } + return 0; +} + +char* +utfrune(const char *s, Rune c) +{ + long c1; + Rune r; + int n; + + if(c < Runesync) /* not part of utf sequence */ + return strchr((char*)s, c); + + for(;;) { + c1 = *(unsigned char*)s; + if(c1 < Runeself) { /* one byte rune */ + if(c1 == 0) + return 0; + if(c1 == c) + return (char*)s; + s++; + continue; + } + n = chartorune(&r, s); + if(r == c) + return (char*)s; + s += n; + } + return 0; +} + +} // namespace re2 diff --git a/extern/re2/util/sparse_array.h b/extern/re2/util/sparse_array.h new file mode 100644 index 0000000000..c81c9f355f --- /dev/null +++ b/extern/re2/util/sparse_array.h @@ -0,0 +1,392 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef UTIL_SPARSE_ARRAY_H_ +#define UTIL_SPARSE_ARRAY_H_ + +// DESCRIPTION +// +// SparseArray(m) is a map from integers in [0, m) to T values. +// It requires (sizeof(T)+sizeof(int))*m memory, but it provides +// fast iteration through the elements in the array and fast clearing +// of the array. The array has a concept of certain elements being +// uninitialized (having no value). +// +// Insertion and deletion are constant time operations. +// +// Allocating the array is a constant time operation +// when memory allocation is a constant time operation. +// +// Clearing the array is a constant time operation (unusual!). +// +// Iterating through the array is an O(n) operation, where n +// is the number of items in the array (not O(m)). +// +// The array iterator visits entries in the order they were first +// inserted into the array. It is safe to add items to the array while +// using an iterator: the iterator will visit indices added to the array +// during the iteration, but will not re-visit indices whose values +// change after visiting. Thus SparseArray can be a convenient +// implementation of a work queue. +// +// The SparseArray implementation is NOT thread-safe. It is up to the +// caller to make sure only one thread is accessing the array. (Typically +// these arrays are temporary values and used in situations where speed is +// important.) +// +// The SparseArray interface does not present all the usual STL bells and +// whistles. +// +// Implemented with reference to Briggs & Torczon, An Efficient +// Representation for Sparse Sets, ACM Letters on Programming Languages +// and Systems, Volume 2, Issue 1-4 (March-Dec. 1993), pp. 59-69. +// +// Briggs & Torczon popularized this technique, but it had been known +// long before their paper. They point out that Aho, Hopcroft, and +// Ullman's 1974 Design and Analysis of Computer Algorithms and Bentley's +// 1986 Programming Pearls both hint at the technique in exercises to the +// reader (in Aho & Hopcroft, exercise 2.12; in Bentley, column 1 +// exercise 8). +// +// Briggs & Torczon describe a sparse set implementation. I have +// trivially generalized it to create a sparse array (actually the original +// target of the AHU and Bentley exercises). + +// IMPLEMENTATION +// +// SparseArray is an array dense_ and an array sparse_ of identical size. +// At any point, the number of elements in the sparse array is size_. +// +// The array dense_ contains the size_ elements in the sparse array (with +// their indices), +// in the order that the elements were first inserted. This array is dense: +// the size_ pairs are dense_[0] through dense_[size_-1]. +// +// The array sparse_ maps from indices in [0,m) to indices in [0,size_). +// For indices present in the array, dense_[sparse_[i]].index_ == i. +// For indices not present in the array, sparse_ can contain any value at all, +// perhaps outside the range [0, size_) but perhaps not. +// +// The lax requirement on sparse_ values makes clearing the array very easy: +// set size_ to 0. Lookups are slightly more complicated. +// An index i has a value in the array if and only if: +// sparse_[i] is in [0, size_) AND +// dense_[sparse_[i]].index_ == i. +// If both these properties hold, only then it is safe to refer to +// dense_[sparse_[i]].value_ +// as the value associated with index i. +// +// To insert a new entry, set sparse_[i] to size_, +// initialize dense_[size_], and then increment size_. +// +// To make the sparse array as efficient as possible for non-primitive types, +// elements may or may not be destroyed when they are deleted from the sparse +// array through a call to resize(). They immediately become inaccessible, but +// they are only guaranteed to be destroyed when the SparseArray destructor is +// called. +// +// A moved-from SparseArray will be empty. + +// Doing this simplifies the logic below. +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#include +#include +#if __has_feature(memory_sanitizer) +#include +#endif +#include +#include +#include + +#include "util/pod_array.h" + +namespace re2 { + +template +class SparseArray { + public: + SparseArray(); + explicit SparseArray(int max_size); + ~SparseArray(); + + // IndexValue pairs: exposed in SparseArray::iterator. + class IndexValue; + + typedef IndexValue* iterator; + typedef const IndexValue* const_iterator; + + SparseArray(const SparseArray& src); + SparseArray(SparseArray&& src); + + SparseArray& operator=(const SparseArray& src); + SparseArray& operator=(SparseArray&& src); + + // Return the number of entries in the array. + int size() const { + return size_; + } + + // Indicate whether the array is empty. + int empty() const { + return size_ == 0; + } + + // Iterate over the array. + iterator begin() { + return dense_.data(); + } + iterator end() { + return dense_.data() + size_; + } + + const_iterator begin() const { + return dense_.data(); + } + const_iterator end() const { + return dense_.data() + size_; + } + + // Change the maximum size of the array. + // Invalidates all iterators. + void resize(int new_max_size); + + // Return the maximum size of the array. + // Indices can be in the range [0, max_size). + int max_size() const { + if (dense_.data() != NULL) + return dense_.size(); + else + return 0; + } + + // Clear the array. + void clear() { + size_ = 0; + } + + // Check whether index i is in the array. + bool has_index(int i) const; + + // Comparison function for sorting. + // Can sort the sparse array so that future iterations + // will visit indices in increasing order using + // std::sort(arr.begin(), arr.end(), arr.less); + static bool less(const IndexValue& a, const IndexValue& b); + + public: + // Set the value at index i to v. + iterator set(int i, const Value& v) { + return SetInternal(true, i, v); + } + + // Set the value at new index i to v. + // Fast but unsafe: only use if has_index(i) is false. + iterator set_new(int i, const Value& v) { + return SetInternal(false, i, v); + } + + // Set the value at index i to v. + // Fast but unsafe: only use if has_index(i) is true. + iterator set_existing(int i, const Value& v) { + return SetExistingInternal(i, v); + } + + // Get the value at index i. + // Fast but unsafe: only use if has_index(i) is true. + Value& get_existing(int i) { + assert(has_index(i)); + return dense_[sparse_[i]].value_; + } + const Value& get_existing(int i) const { + assert(has_index(i)); + return dense_[sparse_[i]].value_; + } + + private: + iterator SetInternal(bool allow_existing, int i, const Value& v) { + DebugCheckInvariants(); + if (static_cast(i) >= static_cast(max_size())) { + assert(false && "illegal index"); + // Semantically, end() would be better here, but we already know + // the user did something stupid, so begin() insulates them from + // dereferencing an invalid pointer. + return begin(); + } + if (!allow_existing) { + assert(!has_index(i)); + create_index(i); + } else { + if (!has_index(i)) + create_index(i); + } + return SetExistingInternal(i, v); + } + + iterator SetExistingInternal(int i, const Value& v) { + DebugCheckInvariants(); + assert(has_index(i)); + dense_[sparse_[i]].value_ = v; + DebugCheckInvariants(); + return dense_.data() + sparse_[i]; + } + + // Add the index i to the array. + // Only use if has_index(i) is known to be false. + // Since it doesn't set the value associated with i, + // this function is private, only intended as a helper + // for other methods. + void create_index(int i); + + // In debug mode, verify that some invariant properties of the class + // are being maintained. This is called at the end of the constructor + // and at the beginning and end of all public non-const member functions. + void DebugCheckInvariants() const; + + // Initializes memory for elements [min, max). + void MaybeInitializeMemory(int min, int max) { +#if __has_feature(memory_sanitizer) + __msan_unpoison(sparse_.data() + min, (max - min) * sizeof sparse_[0]); +#elif defined(RE2_ON_VALGRIND) + for (int i = min; i < max; i++) { + sparse_[i] = 0xababababU; + } +#endif + } + + int size_ = 0; + PODArray sparse_; + PODArray dense_; +}; + +template +SparseArray::SparseArray() = default; + +template +SparseArray::SparseArray(const SparseArray& src) + : size_(src.size_), + sparse_(src.max_size()), + dense_(src.max_size()) { + std::copy_n(src.sparse_.data(), src.max_size(), sparse_.data()); + std::copy_n(src.dense_.data(), src.max_size(), dense_.data()); +} + +template +SparseArray::SparseArray(SparseArray&& src) + : size_(src.size_), + sparse_(std::move(src.sparse_)), + dense_(std::move(src.dense_)) { + src.size_ = 0; +} + +template +SparseArray& SparseArray::operator=(const SparseArray& src) { + // Construct these first for exception safety. + PODArray a(src.max_size()); + PODArray b(src.max_size()); + + size_ = src.size_; + sparse_ = std::move(a); + dense_ = std::move(b); + std::copy_n(src.sparse_.data(), src.max_size(), sparse_.data()); + std::copy_n(src.dense_.data(), src.max_size(), dense_.data()); + return *this; +} + +template +SparseArray& SparseArray::operator=(SparseArray&& src) { + size_ = src.size_; + sparse_ = std::move(src.sparse_); + dense_ = std::move(src.dense_); + src.size_ = 0; + return *this; +} + +// IndexValue pairs: exposed in SparseArray::iterator. +template +class SparseArray::IndexValue { + public: + int index() const { return index_; } + Value& value() { return value_; } + const Value& value() const { return value_; } + + private: + friend class SparseArray; + int index_; + Value value_; +}; + +// Change the maximum size of the array. +// Invalidates all iterators. +template +void SparseArray::resize(int new_max_size) { + DebugCheckInvariants(); + if (new_max_size > max_size()) { + const int old_max_size = max_size(); + + // Construct these first for exception safety. + PODArray a(new_max_size); + PODArray b(new_max_size); + + std::copy_n(sparse_.data(), old_max_size, a.data()); + std::copy_n(dense_.data(), old_max_size, b.data()); + + sparse_ = std::move(a); + dense_ = std::move(b); + + MaybeInitializeMemory(old_max_size, new_max_size); + } + if (size_ > new_max_size) + size_ = new_max_size; + DebugCheckInvariants(); +} + +// Check whether index i is in the array. +template +bool SparseArray::has_index(int i) const { + assert(i >= 0); + assert(i < max_size()); + if (static_cast(i) >= static_cast(max_size())) { + return false; + } + // Unsigned comparison avoids checking sparse_[i] < 0. + return (uint32_t)sparse_[i] < (uint32_t)size_ && + dense_[sparse_[i]].index_ == i; +} + +template +void SparseArray::create_index(int i) { + assert(!has_index(i)); + assert(size_ < max_size()); + sparse_[i] = size_; + dense_[size_].index_ = i; + size_++; +} + +template SparseArray::SparseArray(int max_size) : + sparse_(max_size), dense_(max_size) { + MaybeInitializeMemory(size_, max_size); + DebugCheckInvariants(); +} + +template SparseArray::~SparseArray() { + DebugCheckInvariants(); +} + +template void SparseArray::DebugCheckInvariants() const { + assert(0 <= size_); + assert(size_ <= max_size()); +} + +// Comparison function for sorting. +template bool SparseArray::less(const IndexValue& a, + const IndexValue& b) { + return a.index_ < b.index_; +} + +} // namespace re2 + +#endif // UTIL_SPARSE_ARRAY_H_ diff --git a/extern/re2/util/sparse_set.h b/extern/re2/util/sparse_set.h new file mode 100644 index 0000000000..0d5ad51149 --- /dev/null +++ b/extern/re2/util/sparse_set.h @@ -0,0 +1,264 @@ +// Copyright 2006 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef UTIL_SPARSE_SET_H_ +#define UTIL_SPARSE_SET_H_ + +// DESCRIPTION +// +// SparseSet(m) is a set of integers in [0, m). +// It requires sizeof(int)*m memory, but it provides +// fast iteration through the elements in the set and fast clearing +// of the set. +// +// Insertion and deletion are constant time operations. +// +// Allocating the set is a constant time operation +// when memory allocation is a constant time operation. +// +// Clearing the set is a constant time operation (unusual!). +// +// Iterating through the set is an O(n) operation, where n +// is the number of items in the set (not O(m)). +// +// The set iterator visits entries in the order they were first +// inserted into the set. It is safe to add items to the set while +// using an iterator: the iterator will visit indices added to the set +// during the iteration, but will not re-visit indices whose values +// change after visiting. Thus SparseSet can be a convenient +// implementation of a work queue. +// +// The SparseSet implementation is NOT thread-safe. It is up to the +// caller to make sure only one thread is accessing the set. (Typically +// these sets are temporary values and used in situations where speed is +// important.) +// +// The SparseSet interface does not present all the usual STL bells and +// whistles. +// +// Implemented with reference to Briggs & Torczon, An Efficient +// Representation for Sparse Sets, ACM Letters on Programming Languages +// and Systems, Volume 2, Issue 1-4 (March-Dec. 1993), pp. 59-69. +// +// This is a specialization of sparse array; see sparse_array.h. + +// IMPLEMENTATION +// +// See sparse_array.h for implementation details. + +// Doing this simplifies the logic below. +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#include +#include +#if __has_feature(memory_sanitizer) +#include +#endif +#include +#include +#include + +#include "util/pod_array.h" + +namespace re2 { + +template +class SparseSetT { + public: + SparseSetT(); + explicit SparseSetT(int max_size); + ~SparseSetT(); + + typedef int* iterator; + typedef const int* const_iterator; + + // Return the number of entries in the set. + int size() const { + return size_; + } + + // Indicate whether the set is empty. + int empty() const { + return size_ == 0; + } + + // Iterate over the set. + iterator begin() { + return dense_.data(); + } + iterator end() { + return dense_.data() + size_; + } + + const_iterator begin() const { + return dense_.data(); + } + const_iterator end() const { + return dense_.data() + size_; + } + + // Change the maximum size of the set. + // Invalidates all iterators. + void resize(int new_max_size); + + // Return the maximum size of the set. + // Indices can be in the range [0, max_size). + int max_size() const { + if (dense_.data() != NULL) + return dense_.size(); + else + return 0; + } + + // Clear the set. + void clear() { + size_ = 0; + } + + // Check whether index i is in the set. + bool contains(int i) const; + + // Comparison function for sorting. + // Can sort the sparse set so that future iterations + // will visit indices in increasing order using + // std::sort(arr.begin(), arr.end(), arr.less); + static bool less(int a, int b); + + public: + // Insert index i into the set. + iterator insert(int i) { + return InsertInternal(true, i); + } + + // Insert index i into the set. + // Fast but unsafe: only use if contains(i) is false. + iterator insert_new(int i) { + return InsertInternal(false, i); + } + + private: + iterator InsertInternal(bool allow_existing, int i) { + DebugCheckInvariants(); + if (static_cast(i) >= static_cast(max_size())) { + assert(false && "illegal index"); + // Semantically, end() would be better here, but we already know + // the user did something stupid, so begin() insulates them from + // dereferencing an invalid pointer. + return begin(); + } + if (!allow_existing) { + assert(!contains(i)); + create_index(i); + } else { + if (!contains(i)) + create_index(i); + } + DebugCheckInvariants(); + return dense_.data() + sparse_[i]; + } + + // Add the index i to the set. + // Only use if contains(i) is known to be false. + // This function is private, only intended as a helper + // for other methods. + void create_index(int i); + + // In debug mode, verify that some invariant properties of the class + // are being maintained. This is called at the end of the constructor + // and at the beginning and end of all public non-const member functions. + void DebugCheckInvariants() const; + + // Initializes memory for elements [min, max). + void MaybeInitializeMemory(int min, int max) { +#if __has_feature(memory_sanitizer) + __msan_unpoison(sparse_.data() + min, (max - min) * sizeof sparse_[0]); +#elif defined(RE2_ON_VALGRIND) + for (int i = min; i < max; i++) { + sparse_[i] = 0xababababU; + } +#endif + } + + int size_ = 0; + PODArray sparse_; + PODArray dense_; +}; + +template +SparseSetT::SparseSetT() = default; + +// Change the maximum size of the set. +// Invalidates all iterators. +template +void SparseSetT::resize(int new_max_size) { + DebugCheckInvariants(); + if (new_max_size > max_size()) { + const int old_max_size = max_size(); + + // Construct these first for exception safety. + PODArray a(new_max_size); + PODArray b(new_max_size); + + std::copy_n(sparse_.data(), old_max_size, a.data()); + std::copy_n(dense_.data(), old_max_size, b.data()); + + sparse_ = std::move(a); + dense_ = std::move(b); + + MaybeInitializeMemory(old_max_size, new_max_size); + } + if (size_ > new_max_size) + size_ = new_max_size; + DebugCheckInvariants(); +} + +// Check whether index i is in the set. +template +bool SparseSetT::contains(int i) const { + assert(i >= 0); + assert(i < max_size()); + if (static_cast(i) >= static_cast(max_size())) { + return false; + } + // Unsigned comparison avoids checking sparse_[i] < 0. + return (uint32_t)sparse_[i] < (uint32_t)size_ && + dense_[sparse_[i]] == i; +} + +template +void SparseSetT::create_index(int i) { + assert(!contains(i)); + assert(size_ < max_size()); + sparse_[i] = size_; + dense_[size_] = i; + size_++; +} + +template SparseSetT::SparseSetT(int max_size) : + sparse_(max_size), dense_(max_size) { + MaybeInitializeMemory(size_, max_size); + DebugCheckInvariants(); +} + +template SparseSetT::~SparseSetT() { + DebugCheckInvariants(); +} + +template void SparseSetT::DebugCheckInvariants() const { + assert(0 <= size_); + assert(size_ <= max_size()); +} + +// Comparison function for sorting. +template bool SparseSetT::less(int a, int b) { + return a < b; +} + +typedef SparseSetT SparseSet; + +} // namespace re2 + +#endif // UTIL_SPARSE_SET_H_ diff --git a/extern/re2/util/strutil.cc b/extern/re2/util/strutil.cc new file mode 100644 index 0000000000..fb7e6b1b0c --- /dev/null +++ b/extern/re2/util/strutil.cc @@ -0,0 +1,149 @@ +// Copyright 1999-2005 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include + +#include "util/strutil.h" + +#ifdef _WIN32 +#define snprintf _snprintf +#define vsnprintf _vsnprintf +#endif + +namespace re2 { + +// ---------------------------------------------------------------------- +// CEscapeString() +// Copies 'src' to 'dest', escaping dangerous characters using +// C-style escape sequences. 'src' and 'dest' should not overlap. +// Returns the number of bytes written to 'dest' (not including the \0) +// or (size_t)-1 if there was insufficient space. +// ---------------------------------------------------------------------- +static size_t CEscapeString(const char* src, size_t src_len, + char* dest, size_t dest_len) { + const char* src_end = src + src_len; + size_t used = 0; + + for (; src < src_end; src++) { + if (dest_len - used < 2) // space for two-character escape + return (size_t)-1; + + unsigned char c = *src; + switch (c) { + case '\n': dest[used++] = '\\'; dest[used++] = 'n'; break; + case '\r': dest[used++] = '\\'; dest[used++] = 'r'; break; + case '\t': dest[used++] = '\\'; dest[used++] = 't'; break; + case '\"': dest[used++] = '\\'; dest[used++] = '\"'; break; + case '\'': dest[used++] = '\\'; dest[used++] = '\''; break; + case '\\': dest[used++] = '\\'; dest[used++] = '\\'; break; + default: + // Note that if we emit \xNN and the src character after that is a hex + // digit then that digit must be escaped too to prevent it being + // interpreted as part of the character code by C. + if (c < ' ' || c > '~') { + if (dest_len - used < 5) // space for four-character escape + \0 + return (size_t)-1; + snprintf(dest + used, 5, "\\%03o", c); + used += 4; + } else { + dest[used++] = c; break; + } + } + } + + if (dest_len - used < 1) // make sure that there is room for \0 + return (size_t)-1; + + dest[used] = '\0'; // doesn't count towards return value though + return used; +} + +// ---------------------------------------------------------------------- +// CEscape() +// Copies 'src' to result, escaping dangerous characters using +// C-style escape sequences. 'src' and 'dest' should not overlap. +// ---------------------------------------------------------------------- +std::string CEscape(const StringPiece& src) { + const size_t dest_len = src.size() * 4 + 1; // Maximum possible expansion + char* dest = new char[dest_len]; + const size_t used = CEscapeString(src.data(), src.size(), + dest, dest_len); + std::string s = std::string(dest, used); + delete[] dest; + return s; +} + +void PrefixSuccessor(std::string* prefix) { + // We can increment the last character in the string and be done + // unless that character is 255, in which case we have to erase the + // last character and increment the previous character, unless that + // is 255, etc. If the string is empty or consists entirely of + // 255's, we just return the empty string. + while (!prefix->empty()) { + char& c = prefix->back(); + if (c == '\xff') { // char literal avoids signed/unsigned. + prefix->pop_back(); + } else { + ++c; + break; + } + } +} + +static void StringAppendV(std::string* dst, const char* format, va_list ap) { + // First try with a small fixed size buffer + char space[1024]; + + // It's possible for methods that use a va_list to invalidate + // the data in it upon use. The fix is to make a copy + // of the structure before using it and use that copy instead. + va_list backup_ap; + va_copy(backup_ap, ap); + int result = vsnprintf(space, sizeof(space), format, backup_ap); + va_end(backup_ap); + + if ((result >= 0) && (static_cast(result) < sizeof(space))) { + // It fit + dst->append(space, result); + return; + } + + // Repeatedly increase buffer size until it fits + int length = sizeof(space); + while (true) { + if (result < 0) { + // Older behavior: just try doubling the buffer size + length *= 2; + } else { + // We need exactly "result+1" characters + length = result+1; + } + char* buf = new char[length]; + + // Restore the va_list before we use it again + va_copy(backup_ap, ap); + result = vsnprintf(buf, length, format, backup_ap); + va_end(backup_ap); + + if ((result >= 0) && (result < length)) { + // It fit + dst->append(buf, result); + delete[] buf; + return; + } + delete[] buf; + } +} + +std::string StringPrintf(const char* format, ...) { + va_list ap; + va_start(ap, format); + std::string result; + StringAppendV(&result, format, ap); + va_end(ap); + return result; +} + +} // namespace re2 diff --git a/extern/re2/util/strutil.h b/extern/re2/util/strutil.h new file mode 100644 index 0000000000..a69908a0dd --- /dev/null +++ b/extern/re2/util/strutil.h @@ -0,0 +1,21 @@ +// Copyright 2016 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef UTIL_STRUTIL_H_ +#define UTIL_STRUTIL_H_ + +#include + +#include "re2/stringpiece.h" +#include "util/util.h" + +namespace re2 { + +std::string CEscape(const StringPiece& src); +void PrefixSuccessor(std::string* prefix); +std::string StringPrintf(const char* format, ...); + +} // namespace re2 + +#endif // UTIL_STRUTIL_H_ diff --git a/extern/re2/util/test.cc b/extern/re2/util/test.cc new file mode 100644 index 0000000000..29c8b41420 --- /dev/null +++ b/extern/re2/util/test.cc @@ -0,0 +1,31 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include + +#include "util/test.h" + +DEFINE_string(test_tmpdir, "/var/tmp", "temp directory"); + +struct Test { + void (*fn)(void); + const char *name; +}; + +static Test tests[10000]; +static int ntests; + +void RegisterTest(void (*fn)(void), const char *name) { + tests[ntests].fn = fn; + tests[ntests++].name = name; +} + +int main(int argc, char** argv) { + for (int i = 0; i < ntests; i++) { + printf("%s\n", tests[i].name); + tests[i].fn(); + } + printf("PASS\n"); + return 0; +} diff --git a/extern/re2/util/test.h b/extern/re2/util/test.h new file mode 100644 index 0000000000..5242e94a9c --- /dev/null +++ b/extern/re2/util/test.h @@ -0,0 +1,58 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef UTIL_TEST_H_ +#define UTIL_TEST_H_ + +#include "util/util.h" +#include "util/flags.h" +#include "util/logging.h" + +#define TEST(x, y) \ + void x##y(void); \ + TestRegisterer r##x##y(x##y, # x "." # y); \ + void x##y(void) + +void RegisterTest(void (*)(void), const char*); + +class TestRegisterer { + public: + TestRegisterer(void (*fn)(void), const char *s) { + RegisterTest(fn, s); + } +}; + +// fatal assertions +#define ASSERT_TRUE CHECK +#define ASSERT_FALSE(x) CHECK(!(x)) +#define ASSERT_EQ CHECK_EQ +#define ASSERT_NE CHECK_NE +#define ASSERT_LT CHECK_LT +#define ASSERT_LE CHECK_LE +#define ASSERT_GT CHECK_GT +#define ASSERT_GE CHECK_GE + +// nonfatal assertions +// TODO(rsc): Do a better job? +#define EXPECT_TRUE CHECK +#define EXPECT_FALSE(x) CHECK(!(x)) +#define EXPECT_EQ CHECK_EQ +#define EXPECT_NE CHECK_NE +#define EXPECT_LT CHECK_LT +#define EXPECT_LE CHECK_LE +#define EXPECT_GT CHECK_GT +#define EXPECT_GE CHECK_GE + +namespace testing { +class MallocCounter { + public: + MallocCounter(int x) {} + static const int THIS_THREAD_ONLY = 0; + long long HeapGrowth() { return 0; } + long long PeakHeapGrowth() { return 0; } + void Reset() {} +}; +} // namespace testing + +#endif // UTIL_TEST_H_ diff --git a/extern/re2/util/utf.h b/extern/re2/util/utf.h new file mode 100644 index 0000000000..85b4297239 --- /dev/null +++ b/extern/re2/util/utf.h @@ -0,0 +1,44 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson. + * Copyright (c) 2002 by Lucent Technologies. + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES MAKE ANY + * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY + * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + * + * This file and rune.cc have been converted to compile as C++ code + * in name space re2. + */ + +#ifndef UTIL_UTF_H_ +#define UTIL_UTF_H_ + +#include + +namespace re2 { + +typedef signed int Rune; /* Code-point values in Unicode 4.0 are 21 bits wide.*/ + +enum +{ + UTFmax = 4, /* maximum bytes per rune */ + Runesync = 0x80, /* cannot represent part of a UTF sequence (<) */ + Runeself = 0x80, /* rune and UTF sequences are the same (<) */ + Runeerror = 0xFFFD, /* decoding error in UTF */ + Runemax = 0x10FFFF, /* maximum rune value */ +}; + +int runetochar(char* s, const Rune* r); +int chartorune(Rune* r, const char* s); +int fullrune(const char* s, int n); +int utflen(const char* s); +char* utfrune(const char*, Rune); + +} // namespace re2 + +#endif // UTIL_UTF_H_ diff --git a/extern/re2/util/util.h b/extern/re2/util/util.h new file mode 100644 index 0000000000..8f3e0d0fe7 --- /dev/null +++ b/extern/re2/util/util.h @@ -0,0 +1,34 @@ +// Copyright 2009 The RE2 Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef UTIL_UTIL_H_ +#define UTIL_UTIL_H_ + +#define arraysize(array) (sizeof(array)/sizeof((array)[0])) + +#ifndef ATTRIBUTE_NORETURN +#if defined(__GNUC__) +#define ATTRIBUTE_NORETURN __attribute__((noreturn)) +#elif defined(_MSC_VER) +#define ATTRIBUTE_NORETURN __declspec(noreturn) +#else +#define ATTRIBUTE_NORETURN +#endif +#endif + +#ifndef FALLTHROUGH_INTENDED +#if defined(__clang__) +#define FALLTHROUGH_INTENDED [[clang::fallthrough]] +#elif defined(__GNUC__) && __GNUC__ >= 7 +#define FALLTHROUGH_INTENDED [[gnu::fallthrough]] +#else +#define FALLTHROUGH_INTENDED do {} while (0) +#endif +#endif + +#ifndef NO_THREAD_SAFETY_ANALYSIS +#define NO_THREAD_SAFETY_ANALYSIS +#endif + +#endif // UTIL_UTIL_H_ From 4a1677750fba6e205524c47fb3de8f90e52cdb63 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sun, 11 Aug 2019 21:33:45 -0300 Subject: [PATCH 003/274] Windows build adjustments. --- builds/win32/make_boot.bat | 14 ++++++++++++++ builds/win32/msvc15/FirebirdCommon.props | 2 +- builds/win32/msvc15/common.vcxproj | 10 ++++++---- builds/win32/msvc15/common.vcxproj.filters | 6 ++++++ builds/win32/msvc15/engine.vcxproj | 1 - builds/win32/msvc15/engine.vcxproj.filters | 3 --- builds/win32/msvc15/fbtrace.vcxproj | 2 -- builds/win32/msvc15/fbtrace.vcxproj.filters | 6 ------ 8 files changed, 27 insertions(+), 17 deletions(-) diff --git a/builds/win32/make_boot.bat b/builds/win32/make_boot.bat index 7de8b1fd15..dd0709d4ad 100644 --- a/builds/win32/make_boot.bat +++ b/builds/win32/make_boot.bat @@ -41,6 +41,9 @@ if "%ERRLEV%"=="1" goto :END call :decNumber if "%ERRLEV%"=="1" goto :END +call :re2 +if "%ERRLEV%"=="1" goto :END + call :zlib if "%ERRLEV%"=="1" goto :END @@ -159,6 +162,17 @@ if errorlevel 1 call :boot2 decNumber_%FB_OBJ_DIR% @call set_build_target.bat %* goto :EOF +::=================== +:: BUILD re2 +:re2 +@echo. +@echo Building re2... +REM FIXME: VS version / arch +@cmake -G "Visual Studio %MSVC_VERSION% 2017 Win64" -S %FB_ROOT_PATH%\extern\re2 -B %FB_ROOT_PATH%\extern\re2\builds\%FB_TARGET_PLATFORM% +@cmake --build %FB_ROOT_PATH%\extern\re2\builds\%FB_TARGET_PLATFORM% --target ALL_BUILD --config Release > re2_Release_%FB_TARGET_PLATFORM%.log +@cmake --build %FB_ROOT_PATH%\extern\re2\builds\%FB_TARGET_PLATFORM% --target ALL_BUILD --config Debug > re2_Debug_%FB_TARGET_PLATFORM%.log +goto :EOF + ::=================== :: Extract zlib :zlib diff --git a/builds/win32/msvc15/FirebirdCommon.props b/builds/win32/msvc15/FirebirdCommon.props index 47b5b641b3..5c24a0063f 100644 --- a/builds/win32/msvc15/FirebirdCommon.props +++ b/builds/win32/msvc15/FirebirdCommon.props @@ -10,7 +10,7 @@ /EHsc- %(AdditionalOptions) true - ../../../src/include;../../../src/include/gen;../../../extern/libtomcrypt/src/headers;../../../extern/libtommath;../../../extern/icu/include;../../../extern/zlib;%(AdditionalIncludeDirectories) + ../../../src/include;../../../src/include/gen;../../../extern/libtomcrypt/src/headers;../../../extern/libtommath;../../../extern/icu/include;../../../extern/zlib;../../../extern/re2;%(AdditionalIncludeDirectories) false diff --git a/builds/win32/msvc15/common.vcxproj b/builds/win32/msvc15/common.vcxproj index b3cf37a6f1..ae4a4fe9d2 100644 --- a/builds/win32/msvc15/common.vcxproj +++ b/builds/win32/msvc15/common.vcxproj @@ -87,6 +87,7 @@ + @@ -198,6 +199,7 @@ + @@ -301,7 +303,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) @@ -317,7 +319,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) @@ -336,7 +338,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) @@ -354,7 +356,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc15/common.vcxproj.filters b/builds/win32/msvc15/common.vcxproj.filters index da6508bfca..0944d0518d 100644 --- a/builds/win32/msvc15/common.vcxproj.filters +++ b/builds/win32/msvc15/common.vcxproj.filters @@ -228,6 +228,9 @@ common + + common + common @@ -566,6 +569,9 @@ headers + + headers + headers diff --git a/builds/win32/msvc15/engine.vcxproj b/builds/win32/msvc15/engine.vcxproj index 356835ec3a..feae375b49 100644 --- a/builds/win32/msvc15/engine.vcxproj +++ b/builds/win32/msvc15/engine.vcxproj @@ -317,7 +317,6 @@ - diff --git a/builds/win32/msvc15/engine.vcxproj.filters b/builds/win32/msvc15/engine.vcxproj.filters index bb75ef49ec..8eac730a06 100644 --- a/builds/win32/msvc15/engine.vcxproj.filters +++ b/builds/win32/msvc15/engine.vcxproj.filters @@ -938,9 +938,6 @@ Header files - - Header files - Header files diff --git a/builds/win32/msvc15/fbtrace.vcxproj b/builds/win32/msvc15/fbtrace.vcxproj index f2b17d429c..4b46cccbe4 100644 --- a/builds/win32/msvc15/fbtrace.vcxproj +++ b/builds/win32/msvc15/fbtrace.vcxproj @@ -183,7 +183,6 @@ - @@ -191,7 +190,6 @@ - diff --git a/builds/win32/msvc15/fbtrace.vcxproj.filters b/builds/win32/msvc15/fbtrace.vcxproj.filters index da6c776679..dc253e7e40 100644 --- a/builds/win32/msvc15/fbtrace.vcxproj.filters +++ b/builds/win32/msvc15/fbtrace.vcxproj.filters @@ -30,9 +30,6 @@ Source Files - - Source Files - @@ -50,9 +47,6 @@ Header Files - - Header Files - From 3d6544d9614aaf717958cf77648aacfefb2fd983 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sun, 11 Aug 2019 22:39:42 -0300 Subject: [PATCH 004/274] Misc. --- src/common/SimilarToRegex.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/SimilarToRegex.h b/src/common/SimilarToRegex.h index e1b2554bb9..dcd307757e 100644 --- a/src/common/SimilarToRegex.h +++ b/src/common/SimilarToRegex.h @@ -18,8 +18,8 @@ * */ -#ifndef COMMON_SIMILAR_TO_REGEX_H -#define COMMON_SIMILAR_TO_REGEX_H +#ifndef FB_COMMON_SIMILAR_TO_REGEX_H +#define FB_COMMON_SIMILAR_TO_REGEX_H #include "firebird.h" #include @@ -72,4 +72,4 @@ private: } // namespace Firebird -#endif // COMMON_SIMILAR_TO_REGEX_H +#endif // FB_COMMON_SIMILAR_TO_REGEX_H From c5a211a33a86bd4e2dafdc348407e8961b8ac7e4 Mon Sep 17 00:00:00 2001 From: hvlad Date: Mon, 12 Aug 2019 20:57:07 +0300 Subject: [PATCH 005/274] Fixed typo --- builds/win32/msvc15/common.vcxproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builds/win32/msvc15/common.vcxproj b/builds/win32/msvc15/common.vcxproj index ae4a4fe9d2..c2ccd9f972 100644 --- a/builds/win32/msvc15/common.vcxproj +++ b/builds/win32/msvc15/common.vcxproj @@ -199,7 +199,7 @@ - + From f1964ffc5f07f268fe103b032c08392d49eb77ec Mon Sep 17 00:00:00 2001 From: hvlad Date: Tue, 13 Aug 2019 00:57:10 +0300 Subject: [PATCH 006/274] 1. cmake supplied with VS 2017 doesn't support switch -B 2. Correct usage of target platform (Win32\x64) --- builds/win32/make_boot.bat | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/builds/win32/make_boot.bat b/builds/win32/make_boot.bat index dd0709d4ad..1ac752568d 100644 --- a/builds/win32/make_boot.bat +++ b/builds/win32/make_boot.bat @@ -167,10 +167,13 @@ goto :EOF :re2 @echo. @echo Building re2... -REM FIXME: VS version / arch -@cmake -G "Visual Studio %MSVC_VERSION% 2017 Win64" -S %FB_ROOT_PATH%\extern\re2 -B %FB_ROOT_PATH%\extern\re2\builds\%FB_TARGET_PLATFORM% +REM FIXME: VS version +@mkdir %FB_ROOT_PATH%\extern\re2\builds\%FB_TARGET_PLATFORM% 2>nul +@pushd %FB_ROOT_PATH%\extern\re2\builds\%FB_TARGET_PLATFORM% +@cmake -G "Visual Studio %MSVC_VERSION% 2017" -A %FB_TARGET_PLATFORM% -S %FB_ROOT_PATH%\extern\re2 @cmake --build %FB_ROOT_PATH%\extern\re2\builds\%FB_TARGET_PLATFORM% --target ALL_BUILD --config Release > re2_Release_%FB_TARGET_PLATFORM%.log @cmake --build %FB_ROOT_PATH%\extern\re2\builds\%FB_TARGET_PLATFORM% --target ALL_BUILD --config Debug > re2_Debug_%FB_TARGET_PLATFORM%.log +@popd goto :EOF ::=================== From 100d7b07dbed66ff7f63b93364db4d0838d6a2d8 Mon Sep 17 00:00:00 2001 From: hvlad Date: Tue, 13 Aug 2019 01:05:50 +0300 Subject: [PATCH 007/274] It is not required to specify VS version --- builds/win32/make_boot.bat | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/builds/win32/make_boot.bat b/builds/win32/make_boot.bat index 1ac752568d..b4d62b90dd 100644 --- a/builds/win32/make_boot.bat +++ b/builds/win32/make_boot.bat @@ -167,10 +167,9 @@ goto :EOF :re2 @echo. @echo Building re2... -REM FIXME: VS version @mkdir %FB_ROOT_PATH%\extern\re2\builds\%FB_TARGET_PLATFORM% 2>nul @pushd %FB_ROOT_PATH%\extern\re2\builds\%FB_TARGET_PLATFORM% -@cmake -G "Visual Studio %MSVC_VERSION% 2017" -A %FB_TARGET_PLATFORM% -S %FB_ROOT_PATH%\extern\re2 +@cmake -G "Visual Studio %MSVC_VERSION%" -A %FB_TARGET_PLATFORM% -S %FB_ROOT_PATH%\extern\re2 @cmake --build %FB_ROOT_PATH%\extern\re2\builds\%FB_TARGET_PLATFORM% --target ALL_BUILD --config Release > re2_Release_%FB_TARGET_PLATFORM%.log @cmake --build %FB_ROOT_PATH%\extern\re2\builds\%FB_TARGET_PLATFORM% --target ALL_BUILD --config Debug > re2_Debug_%FB_TARGET_PLATFORM%.log @popd From 60f9eb851c563d47f39525684c335b89b40c321d Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Tue, 13 Aug 2019 12:25:40 -0300 Subject: [PATCH 008/274] Do not ignore all Makefile's from the tree. --- .gitignore | 2 +- extern/editline/.gitignore | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a91a3e069b..78face02f5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ output_x64/ m4/ autom4te.cache/ builds/install_images -Makefile aclocal.m4 *.patch *.diff @@ -19,3 +18,4 @@ libtool src/dsql/parse.cpp .vscode/.browse.VC.db extern/decNumber/libdecFloat*.a +/Makefile diff --git a/extern/editline/.gitignore b/extern/editline/.gitignore index d318988c67..4a5ee01d0d 100644 --- a/extern/editline/.gitignore +++ b/extern/editline/.gitignore @@ -1,3 +1,4 @@ +Makefile src/*.o src/*.d src/*.a From 19993913004e745ba29388aa53b81a71daf34d43 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Tue, 13 Aug 2019 12:26:12 -0300 Subject: [PATCH 009/274] Added re2 Makefile. --- extern/re2/Makefile | 345 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 extern/re2/Makefile diff --git a/extern/re2/Makefile b/extern/re2/Makefile new file mode 100644 index 0000000000..f001f0640c --- /dev/null +++ b/extern/re2/Makefile @@ -0,0 +1,345 @@ +# Copyright 2009 The RE2 Authors. All Rights Reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# To build against ICU for full Unicode properties support, +# uncomment the next two lines: +# CCICU=$(shell pkg-config icu-uc --cflags) -DRE2_USE_ICU +# LDICU=$(shell pkg-config icu-uc --libs) + +# To build against PCRE for testing or benchmarking, +# uncomment the next two lines: +# CCPCRE=-I/usr/local/include -DUSEPCRE +# LDPCRE=-L/usr/local/lib -lpcre + +CXX?=g++ +# can override +CXXFLAGS?=-O3 -g +LDFLAGS?= +# required +RE2_CXXFLAGS?=-std=c++11 -pthread -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -I. $(CCICU) $(CCPCRE) +RE2_LDFLAGS?=-pthread $(LDICU) $(LDPCRE) +AR?=ar +ARFLAGS?=rsc +NM?=nm +NMFLAGS?=-p + +# Variables mandated by GNU, the arbiter of all good taste on the internet. +# http://www.gnu.org/prep/standards/standards.html +prefix=/usr/local +exec_prefix=$(prefix) +includedir=$(prefix)/include +libdir=$(exec_prefix)/lib +INSTALL=install +INSTALL_DATA=$(INSTALL) -m 644 + +# Work around the weirdness of sed(1) on Darwin. :/ +ifeq ($(shell uname),Darwin) +SED_INPLACE=sed -i '' +else ifeq ($(shell uname),SunOS) +SED_INPLACE=sed -i +else +SED_INPLACE=sed -i +endif + +# ABI version +# http://tldp.org/HOWTO/Program-Library-HOWTO/shared-libraries.html +SONAME=0 + +# To rebuild the Tables generated by Perl and Python scripts (requires Internet +# access for Unicode data), uncomment the following line: +# REBUILD_TABLES=1 + +# The SunOS linker does not support wildcards. :( +ifeq ($(shell uname),Darwin) +SOEXT=dylib +SOEXTVER=$(SONAME).$(SOEXT) +SOEXTVER00=$(SONAME).0.0.$(SOEXT) +MAKE_SHARED_LIBRARY=$(CXX) -dynamiclib -Wl,-install_name,$(libdir)/libre2.$(SOEXTVER),-exported_symbols_list,libre2.symbols.darwin $(RE2_LDFLAGS) $(LDFLAGS) +else ifeq ($(shell uname),SunOS) +SOEXT=so +SOEXTVER=$(SOEXT).$(SONAME) +SOEXTVER00=$(SOEXT).$(SONAME).0.0 +MAKE_SHARED_LIBRARY=$(CXX) -shared -Wl,-soname,libre2.$(SOEXTVER) $(RE2_LDFLAGS) $(LDFLAGS) +else +SOEXT=so +SOEXTVER=$(SOEXT).$(SONAME) +SOEXTVER00=$(SOEXT).$(SONAME).0.0 +MAKE_SHARED_LIBRARY=$(CXX) -shared -Wl,-soname,libre2.$(SOEXTVER),--version-script,libre2.symbols $(RE2_LDFLAGS) $(LDFLAGS) +endif + +all: obj/libre2.a obj/so/libre2.$(SOEXT) + +INSTALL_HFILES=\ + re2/filtered_re2.h\ + re2/re2.h\ + re2/set.h\ + re2/stringpiece.h\ + +HFILES=\ + util/benchmark.h\ + util/flags.h\ + util/logging.h\ + util/mix.h\ + util/mutex.h\ + util/pcre.h\ + util/pod_array.h\ + util/sparse_array.h\ + util/sparse_set.h\ + util/strutil.h\ + util/test.h\ + util/utf.h\ + util/util.h\ + re2/bitmap256.h\ + re2/filtered_re2.h\ + re2/prefilter.h\ + re2/prefilter_tree.h\ + re2/prog.h\ + re2/re2.h\ + re2/regexp.h\ + re2/set.h\ + re2/stringpiece.h\ + re2/testing/exhaustive_tester.h\ + re2/testing/regexp_generator.h\ + re2/testing/string_generator.h\ + re2/testing/tester.h\ + re2/unicode_casefold.h\ + re2/unicode_groups.h\ + re2/walker-inl.h\ + +OFILES=\ + obj/util/rune.o\ + obj/util/strutil.o\ + obj/re2/bitstate.o\ + obj/re2/compile.o\ + obj/re2/dfa.o\ + obj/re2/filtered_re2.o\ + obj/re2/mimics_pcre.o\ + obj/re2/nfa.o\ + obj/re2/onepass.o\ + obj/re2/parse.o\ + obj/re2/perl_groups.o\ + obj/re2/prefilter.o\ + obj/re2/prefilter_tree.o\ + obj/re2/prog.o\ + obj/re2/re2.o\ + obj/re2/regexp.o\ + obj/re2/set.o\ + obj/re2/simplify.o\ + obj/re2/stringpiece.o\ + obj/re2/tostring.o\ + obj/re2/unicode_casefold.o\ + obj/re2/unicode_groups.o\ + +TESTOFILES=\ + obj/util/pcre.o\ + obj/re2/testing/backtrack.o\ + obj/re2/testing/dump.o\ + obj/re2/testing/exhaustive_tester.o\ + obj/re2/testing/null_walker.o\ + obj/re2/testing/regexp_generator.o\ + obj/re2/testing/string_generator.o\ + obj/re2/testing/tester.o\ + +TESTS=\ + obj/test/charclass_test\ + obj/test/compile_test\ + obj/test/filtered_re2_test\ + obj/test/mimics_pcre_test\ + obj/test/parse_test\ + obj/test/possible_match_test\ + obj/test/re2_test\ + obj/test/re2_arg_test\ + obj/test/regexp_test\ + obj/test/required_prefix_test\ + obj/test/search_test\ + obj/test/set_test\ + obj/test/simplify_test\ + obj/test/string_generator_test\ + +BIGTESTS=\ + obj/test/dfa_test\ + obj/test/exhaustive1_test\ + obj/test/exhaustive2_test\ + obj/test/exhaustive3_test\ + obj/test/exhaustive_test\ + obj/test/random_test\ + +SOFILES=$(patsubst obj/%,obj/so/%,$(OFILES)) +# We use TESTOFILES for testing the shared lib, only it is built differently. +STESTS=$(patsubst obj/%,obj/so/%,$(TESTS)) +SBIGTESTS=$(patsubst obj/%,obj/so/%,$(BIGTESTS)) + +DOFILES=$(patsubst obj/%,obj/dbg/%,$(OFILES)) +DTESTOFILES=$(patsubst obj/%,obj/dbg/%,$(TESTOFILES)) +DTESTS=$(patsubst obj/%,obj/dbg/%,$(TESTS)) +DBIGTESTS=$(patsubst obj/%,obj/dbg/%,$(BIGTESTS)) + +obj/%.o: %.cc $(HFILES) + @mkdir -p $$(dirname $@) + $(CXX) -c -o $@ $(CPPFLAGS) $(RE2_CXXFLAGS) $(CXXFLAGS) -DNDEBUG $*.cc + +obj/dbg/%.o: %.cc $(HFILES) + @mkdir -p $$(dirname $@) + $(CXX) -c -o $@ $(CPPFLAGS) $(RE2_CXXFLAGS) $(CXXFLAGS) $*.cc + +obj/so/%.o: %.cc $(HFILES) + @mkdir -p $$(dirname $@) + $(CXX) -c -o $@ -fPIC $(CPPFLAGS) $(RE2_CXXFLAGS) $(CXXFLAGS) -DNDEBUG $*.cc + +obj/libre2.a: $(OFILES) + @mkdir -p obj + $(AR) $(ARFLAGS) obj/libre2.a $(OFILES) + +obj/dbg/libre2.a: $(DOFILES) + @mkdir -p obj/dbg + $(AR) $(ARFLAGS) obj/dbg/libre2.a $(DOFILES) + +obj/so/libre2.$(SOEXT): $(SOFILES) + @mkdir -p obj/so + $(MAKE_SHARED_LIBRARY) -o obj/so/libre2.$(SOEXTVER) $(SOFILES) + ln -sf libre2.$(SOEXTVER) $@ + +obj/dbg/test/%: obj/dbg/libre2.a obj/dbg/re2/testing/%.o $(DTESTOFILES) obj/dbg/util/test.o + @mkdir -p obj/dbg/test + $(CXX) -o $@ obj/dbg/re2/testing/$*.o $(DTESTOFILES) obj/dbg/util/test.o obj/dbg/libre2.a $(RE2_LDFLAGS) $(LDFLAGS) + +obj/test/%: obj/libre2.a obj/re2/testing/%.o $(TESTOFILES) obj/util/test.o + @mkdir -p obj/test + $(CXX) -o $@ obj/re2/testing/$*.o $(TESTOFILES) obj/util/test.o obj/libre2.a $(RE2_LDFLAGS) $(LDFLAGS) + +# Test the shared lib, falling back to the static lib for private symbols +obj/so/test/%: obj/so/libre2.$(SOEXT) obj/libre2.a obj/re2/testing/%.o $(TESTOFILES) obj/util/test.o + @mkdir -p obj/so/test + $(CXX) -o $@ obj/re2/testing/$*.o $(TESTOFILES) obj/util/test.o -Lobj/so -lre2 obj/libre2.a $(RE2_LDFLAGS) $(LDFLAGS) + +obj/test/regexp_benchmark: obj/libre2.a obj/re2/testing/regexp_benchmark.o $(TESTOFILES) obj/util/benchmark.o + @mkdir -p obj/test + $(CXX) -o $@ obj/re2/testing/regexp_benchmark.o $(TESTOFILES) obj/util/benchmark.o obj/libre2.a $(RE2_LDFLAGS) $(LDFLAGS) + +# re2_fuzzer is a target for fuzzers like libFuzzer and AFL. This fake fuzzing +# is simply a way to check that the target builds and then to run it against a +# fixed set of inputs. To perform real fuzzing, refer to the documentation for +# libFuzzer (llvm.org/docs/LibFuzzer.html) and AFL (lcamtuf.coredump.cx/afl/). +obj/test/re2_fuzzer: obj/libre2.a obj/re2/fuzzing/re2_fuzzer.o obj/util/fuzz.o + @mkdir -p obj/test + $(CXX) -o $@ obj/re2/fuzzing/re2_fuzzer.o obj/util/fuzz.o obj/libre2.a $(RE2_LDFLAGS) $(LDFLAGS) + +ifdef REBUILD_TABLES +re2/perl_groups.cc: re2/make_perl_groups.pl + perl $< > $@ + +re2/unicode_%.cc: re2/make_unicode_%.py + python $< > $@ + +.PRECIOUS: re2/perl_groups.cc re2/unicode_casefold.cc re2/unicode_groups.cc +endif + +distclean: clean + rm -f re2/perl_groups.cc re2/unicode_casefold.cc re2/unicode_groups.cc + +clean: + rm -rf obj + rm -f re2/*.pyc + +testofiles: $(TESTOFILES) + +test: $(DTESTS) $(TESTS) $(STESTS) debug-test static-test shared-test + +debug-test: $(DTESTS) + @./runtests $(DTESTS) + +static-test: $(TESTS) + @./runtests $(TESTS) + +shared-test: $(STESTS) + @./runtests -shared-library-path obj/so $(STESTS) + +debug-bigtest: $(DTESTS) $(DBIGTESTS) + @./runtests $(DTESTS) $(DBIGTESTS) + +static-bigtest: $(TESTS) $(BIGTESTS) + @./runtests $(TESTS) $(BIGTESTS) + +shared-bigtest: $(STESTS) $(SBIGTESTS) + @./runtests -shared-library-path obj/so $(STESTS) $(SBIGTESTS) + +benchmark: obj/test/regexp_benchmark + +fuzz: obj/test/re2_fuzzer + +install: obj/libre2.a obj/so/libre2.$(SOEXT) + mkdir -p $(DESTDIR)$(includedir)/re2 $(DESTDIR)$(libdir)/pkgconfig + $(INSTALL_DATA) $(INSTALL_HFILES) $(DESTDIR)$(includedir)/re2 + $(INSTALL) obj/libre2.a $(DESTDIR)$(libdir)/libre2.a + $(INSTALL) obj/so/libre2.$(SOEXT) $(DESTDIR)$(libdir)/libre2.$(SOEXTVER00) + ln -sf libre2.$(SOEXTVER00) $(DESTDIR)$(libdir)/libre2.$(SOEXTVER) + ln -sf libre2.$(SOEXTVER00) $(DESTDIR)$(libdir)/libre2.$(SOEXT) + $(INSTALL_DATA) re2.pc $(DESTDIR)$(libdir)/pkgconfig/re2.pc + $(SED_INPLACE) -e "s#@prefix@#${prefix}#" $(DESTDIR)$(libdir)/pkgconfig/re2.pc + $(SED_INPLACE) -e "s#@exec_prefix@#${exec_prefix}#" $(DESTDIR)$(libdir)/pkgconfig/re2.pc + $(SED_INPLACE) -e "s#@includedir@#${includedir}#" $(DESTDIR)$(libdir)/pkgconfig/re2.pc + $(SED_INPLACE) -e "s#@libdir@#${libdir}#" $(DESTDIR)$(libdir)/pkgconfig/re2.pc + +testinstall: static-testinstall shared-testinstall + @echo + @echo Install tests passed. + @echo + +static-testinstall: CXXFLAGS:=-std=c++11 -pthread -I$(DESTDIR)$(includedir) $(CXXFLAGS) +static-testinstall: LDFLAGS:=-pthread -L$(DESTDIR)$(libdir) -l:libre2.a $(LDICU) $(LDFLAGS) +static-testinstall: + @mkdir -p obj + @cp testinstall.cc obj +ifeq ($(shell uname),Darwin) + @echo Skipping test for libre2.a on Darwin. +else ifeq ($(shell uname),SunOS) + @echo Skipping test for libre2.a on SunOS. +else + (cd obj && $(CXX) testinstall.cc -o testinstall $(CXXFLAGS) $(LDFLAGS)) + obj/testinstall +endif + +shared-testinstall: CXXFLAGS:=-std=c++11 -pthread -I$(DESTDIR)$(includedir) $(CXXFLAGS) +shared-testinstall: LDFLAGS:=-pthread -L$(DESTDIR)$(libdir) -lre2 $(LDICU) $(LDFLAGS) +shared-testinstall: + @mkdir -p obj + @cp testinstall.cc obj + (cd obj && $(CXX) testinstall.cc -o testinstall $(CXXFLAGS) $(LDFLAGS)) +ifeq ($(shell uname),Darwin) + DYLD_LIBRARY_PATH="$(DESTDIR)$(libdir):$(DYLD_LIBRARY_PATH)" obj/testinstall +else + LD_LIBRARY_PATH="$(DESTDIR)$(libdir):$(LD_LIBRARY_PATH)" obj/testinstall +endif + +benchlog: obj/test/regexp_benchmark + (echo '==BENCHMARK==' `hostname` `date`; \ + (uname -a; $(CXX) --version; git rev-parse --short HEAD; file obj/test/regexp_benchmark) | sed 's/^/# /'; \ + echo; \ + ./obj/test/regexp_benchmark 'PCRE|RE2') | tee -a benchlog.$$(hostname | sed 's/\..*//') + +# Keep gmake from deleting intermediate files it creates. +# This makes repeated builds faster and preserves debug info on OS X. + +.PRECIOUS: obj/%.o obj/dbg/%.o obj/so/%.o obj/libre2.a \ + obj/dbg/libre2.a obj/so/libre2.a \ + obj/test/% obj/so/test/% obj/dbg/test/% + +log: + $(MAKE) clean + $(MAKE) CXXFLAGS="$(CXXFLAGS) -DLOGGING=1" \ + $(filter obj/test/exhaustive%_test,$(BIGTESTS)) + echo '#' RE2 exhaustive tests built by make log >re2-exhaustive.txt + echo '#' $$(date) >>re2-exhaustive.txt + obj/test/exhaustive_test |grep -v '^PASS$$' >>re2-exhaustive.txt + obj/test/exhaustive1_test |grep -v '^PASS$$' >>re2-exhaustive.txt + obj/test/exhaustive2_test |grep -v '^PASS$$' >>re2-exhaustive.txt + obj/test/exhaustive3_test |grep -v '^PASS$$' >>re2-exhaustive.txt + + $(MAKE) CXXFLAGS="$(CXXFLAGS) -DLOGGING=1" obj/test/search_test + echo '#' RE2 basic search tests built by make $@ >re2-search.txt + echo '#' $$(date) >>re2-search.txt + obj/test/search_test |grep -v '^PASS$$' >>re2-search.txt + +x: x.cc obj/libre2.a + g++ -I. -o x x.cc obj/libre2.a From 3d47c2224c255f5cc05495628963ebb32c88b622 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Tue, 13 Aug 2019 13:22:18 -0300 Subject: [PATCH 010/274] Adjust posix build. --- builds/posix/Makefile.in | 3 +++ builds/posix/make.defaults | 8 ++++---- builds/posix/make.rules | 2 +- src/common/SimilarToRegex.h | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/builds/posix/Makefile.in b/builds/posix/Makefile.in index 61f59c7bf2..c34bca0b7e 100644 --- a/builds/posix/Makefile.in +++ b/builds/posix/Makefile.in @@ -147,6 +147,9 @@ external: $(MAKE) -C $(ROOT)/extern/decNumber ln -sf $(ROOT)/extern/decNumber/libdecFloat.a $(LIB) + CXXFLAGS="-O3 -g -fPIC" $(MAKE) -C $(ROOT)/extern/re2 + ln -sf $(ROOT)/extern/re2/obj/libre2.a $(LIB) + ifeq ($(TOMMATH_BUILD_FLG),Y) CFLAGS="$(CFLAGS)" $(MAKE) -C $(ROOT)/extern/libtommath -f makefile.shared GCC=$(GCC) ln -sf $(TOMMATH_SO).$(TOMMATH_VER) $(LIB) diff --git a/builds/posix/make.defaults b/builds/posix/make.defaults index 8351fc4579..46712bb61f 100755 --- a/builds/posix/make.defaults +++ b/builds/posix/make.defaults @@ -134,6 +134,7 @@ CAS_OPTIONS=@CAS_OPTIONS@ # multiple-precision integer library MATHLIB=@MATHLIB@ DECLIB=-ldecFloat$(CROSS) +RE2CLIB=-lre2 # crypt library CRYPTLIB=@CRYPTLIB@ @@ -182,8 +183,8 @@ STATICLIB_LINK = $(AR) crus EXE_LINK = $(CXX) $(GLOB_OPTIONS) $(CXXFLAGS) STATICEXE_LINK = $(CXX) $(GLOB_OPTIONS) $(CXXFLAGS) -LINK_LIBS = @LIBS@ $(DECLIB) -SO_LINK_LIBS = @LIBS@ $(DECLIB) +LINK_LIBS = @LIBS@ $(DECLIB) $(RE2CLIB) +SO_LINK_LIBS = @LIBS@ $(DECLIB) $(RE2CLIB) # Default extensions @@ -307,8 +308,7 @@ endif LIB_PATH_OPTS = $(call LIB_LINK_RPATH,lib) $(call LIB_LINK_RPATH,intl) LIB_LINK_SONAME= -Wl,-soname,$(1) LIB_LINK_MAPFILE= -Wl,--version-script,$(1) -# FIXME: -FIREBIRD_LIBRARY_LINK= -L$(LIB) -lfbclient $(MATHLIB) $(CRYPTLIB) -lre2 +FIREBIRD_LIBRARY_LINK= -L$(LIB) -lfbclient $(MATHLIB) $(CRYPTLIB) EXE_LINK_OPTIONS= $(LDFLAGS) $(THR_FLAGS) $(UNDEF_FLAGS) $(LIB_PATH_OPTS) $(LINK_EMPTY_SYMBOLS) LIB_LINK_OPTIONS= $(LDFLAGS) $(THR_FLAGS) -shared diff --git a/builds/posix/make.rules b/builds/posix/make.rules index 7b1483058d..8b48ea50bd 100644 --- a/builds/posix/make.rules +++ b/builds/posix/make.rules @@ -32,7 +32,7 @@ # Please don't use compiler/platform specific flags here - nmcc 02-Nov-2002 -WFLAGS =-I$(SRC_ROOT)/include/gen -I$(SRC_ROOT)/include $(CPPFLAGS) +WFLAGS =-I$(SRC_ROOT)/include/gen -I$(SRC_ROOT)/include -I$(ROOT)/extern/re2 $(CPPFLAGS) ifeq ($(TOMMATH_BUILD_FLG),Y) WFLAGS += -I$(TOMMATH_INC) diff --git a/src/common/SimilarToRegex.h b/src/common/SimilarToRegex.h index dcd307757e..1560b7104e 100644 --- a/src/common/SimilarToRegex.h +++ b/src/common/SimilarToRegex.h @@ -22,7 +22,7 @@ #define FB_COMMON_SIMILAR_TO_REGEX_H #include "firebird.h" -#include +#include "re2/re2.h" #include "../common/classes/auto.h" #include "../common/classes/array.h" #include "../common/classes/fb_string.h" From bb6e08c7d1bde58d93d95c2dcacd700a08517261 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Tue, 13 Aug 2019 22:31:10 -0300 Subject: [PATCH 011/274] Misc. --- builds/posix/make.defaults | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builds/posix/make.defaults b/builds/posix/make.defaults index 46712bb61f..2ed9d328f8 100755 --- a/builds/posix/make.defaults +++ b/builds/posix/make.defaults @@ -134,7 +134,7 @@ CAS_OPTIONS=@CAS_OPTIONS@ # multiple-precision integer library MATHLIB=@MATHLIB@ DECLIB=-ldecFloat$(CROSS) -RE2CLIB=-lre2 +RE2LIB=-lre2 # crypt library CRYPTLIB=@CRYPTLIB@ @@ -183,8 +183,8 @@ STATICLIB_LINK = $(AR) crus EXE_LINK = $(CXX) $(GLOB_OPTIONS) $(CXXFLAGS) STATICEXE_LINK = $(CXX) $(GLOB_OPTIONS) $(CXXFLAGS) -LINK_LIBS = @LIBS@ $(DECLIB) $(RE2CLIB) -SO_LINK_LIBS = @LIBS@ $(DECLIB) $(RE2CLIB) +LINK_LIBS = @LIBS@ $(DECLIB) $(RE2LIB) +SO_LINK_LIBS = @LIBS@ $(DECLIB) $(RE2LIB) # Default extensions From 084b3ee1fca4b380ae076cb081d1e3f78a2c5095 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Mon, 26 Aug 2019 21:47:44 -0300 Subject: [PATCH 012/274] Simplification. --- src/common/SimilarToRegex.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/common/SimilarToRegex.cpp b/src/common/SimilarToRegex.cpp index 1d49478918..d4ed5814f5 100644 --- a/src/common/SimilarToRegex.cpp +++ b/src/common/SimilarToRegex.cpp @@ -54,9 +54,6 @@ namespace status_exception::raise(Arg::Gds(isc_escape_invalid)); } - if (flags & FLAG_CASE_INSENSITIVE) - re2PatternStr.append("(?i)"); - if (flags & FLAG_GROUP_CAPTURE) re2PatternStr.append("("); @@ -73,6 +70,7 @@ namespace RE2::Options options; options.set_log_errors(false); options.set_dot_nl(true); + options.set_case_sensitive(!(flags & FLAG_CASE_INSENSITIVE)); re2::StringPiece sp((const char*) re2PatternStr.c_str(), re2PatternStr.length()); regexp = FB_NEW_POOL(pool) RE2(sp, options); @@ -662,13 +660,13 @@ namespace AutoPtr regexp1, regexp2, regexp3; - SimilarToCompiler compiler1(pool, regexp1, (flags & FLAG_CASE_INSENSITIVE) | FLAG_PREFER_FEWER, + SimilarToCompiler compiler1(pool, regexp1, FLAG_PREFER_FEWER, aPatternStr, positions[0] - escapeLen - 1, escapeStr, escapeLen); - SimilarToCompiler compiler2(pool, regexp2, (flags & FLAG_CASE_INSENSITIVE), + SimilarToCompiler compiler2(pool, regexp2, 0, aPatternStr + positions[0], positions[1] - positions[0] - escapeLen - 1, escapeStr, escapeLen); - SimilarToCompiler compiler3(pool, regexp3, (flags & FLAG_CASE_INSENSITIVE) | FLAG_PREFER_FEWER, + SimilarToCompiler compiler3(pool, regexp3, FLAG_PREFER_FEWER, aPatternStr + positions[1], patternLen - positions[1], escapeStr, escapeLen); string finalRe2Pattern; @@ -693,6 +691,7 @@ namespace RE2::Options options; options.set_log_errors(false); options.set_dot_nl(true); + options.set_case_sensitive(!(flags & FLAG_CASE_INSENSITIVE)); re2::StringPiece sp((const char*) finalRe2Pattern.c_str(), finalRe2Pattern.length()); regexp = FB_NEW_POOL(pool) RE2(sp, options); From 0969fb70e8bc477bb0faae4f482f636c2abe8fdf Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Mon, 26 Aug 2019 21:47:44 -0300 Subject: [PATCH 013/274] Use -static-libstdc++ to avoid crashes. --- builds/posix/make.defaults | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builds/posix/make.defaults b/builds/posix/make.defaults index 2ed9d328f8..b0ff592706 100755 --- a/builds/posix/make.defaults +++ b/builds/posix/make.defaults @@ -178,10 +178,10 @@ AC_CXXFLAGS = @CXXFLAGS@ # LINKER COMMANDS -LIB_LINK = $(CXX) $(GLOB_OPTIONS) $(CXXFLAGS) +LIB_LINK = $(CXX) $(GLOB_OPTIONS) $(CXXFLAGS) -static-libstdc++ STATICLIB_LINK = $(AR) crus -EXE_LINK = $(CXX) $(GLOB_OPTIONS) $(CXXFLAGS) -STATICEXE_LINK = $(CXX) $(GLOB_OPTIONS) $(CXXFLAGS) +EXE_LINK = $(CXX) $(GLOB_OPTIONS) $(CXXFLAGS) -static-libstdc++ +STATICEXE_LINK = $(CXX) $(GLOB_OPTIONS) $(CXXFLAGS) -static-libstdc++ LINK_LIBS = @LIBS@ $(DECLIB) $(RE2LIB) SO_LINK_LIBS = @LIBS@ $(DECLIB) $(RE2LIB) From ff06569f127caf3128a5b3290f67eb468f86fe8d Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Mon, 26 Aug 2019 21:47:44 -0300 Subject: [PATCH 014/274] Add finalizers to free memory or other resources not managed by memory pool. --- src/common/SimilarToRegex.cpp | 24 +++++++++++++ src/common/SimilarToRegex.h | 10 ++++++ src/common/classes/alloc.cpp | 63 +++++++++++++++++++++++++++++++++++ src/common/classes/alloc.h | 21 ++++++++++++ 4 files changed, 118 insertions(+) diff --git a/src/common/SimilarToRegex.cpp b/src/common/SimilarToRegex.cpp index d4ed5814f5..c32f55499d 100644 --- a/src/common/SimilarToRegex.cpp +++ b/src/common/SimilarToRegex.cpp @@ -739,6 +739,18 @@ SimilarToRegex::SimilarToRegex(MemoryPool& pool, bool caseInsensitive, SimilarToCompiler compiler(pool, regexp, FLAG_GROUP_CAPTURE | FLAG_PREFER_FEWER | (caseInsensitive ? FLAG_CASE_INSENSITIVE : 0), patternStr, patternLen, escapeStr, escapeLen); + + finalizerToken = pool.registerFinalizer(finalizer, this); +} + +SimilarToRegex::~SimilarToRegex() +{ + getPool().unregisterFinalizer(finalizerToken); +} + +void SimilarToRegex::finalizer(SimilarToRegex* self) +{ + self->regexp.reset(); } bool SimilarToRegex::matches(const char* buffer, unsigned bufferLen, Array* matchPosArray) @@ -797,6 +809,18 @@ SubstringSimilarRegex::SubstringSimilarRegex(MemoryPool& pool, bool caseInsensit SubstringSimilarCompiler compiler(pool, regexp, (caseInsensitive ? FLAG_CASE_INSENSITIVE : 0), patternStr, patternLen, escapeStr, escapeLen); + + finalizerToken = pool.registerFinalizer(finalizer, this); +} + +SubstringSimilarRegex::~SubstringSimilarRegex() +{ + getPool().unregisterFinalizer(finalizerToken); +} + +void SubstringSimilarRegex::finalizer(SubstringSimilarRegex* self) +{ + self->regexp.reset(); } bool SubstringSimilarRegex::matches(const char* buffer, unsigned bufferLen, diff --git a/src/common/SimilarToRegex.h b/src/common/SimilarToRegex.h index 1560b7104e..8244eaf1ad 100644 --- a/src/common/SimilarToRegex.h +++ b/src/common/SimilarToRegex.h @@ -43,11 +43,16 @@ public: public: SimilarToRegex(MemoryPool& pool, bool caseInsensitive, const char* patternStr, unsigned patternLen, const char* escapeStr, unsigned escapeLen); + ~SimilarToRegex(); + +private: + static void finalizer(SimilarToRegex* self); public: bool matches(const char* buffer, unsigned bufferLen, Array* matchPosArray = nullptr); private: + void* finalizerToken = nullptr; AutoPtr regexp; }; @@ -61,11 +66,16 @@ class SubstringSimilarRegex : public PermanentStorage public: SubstringSimilarRegex(MemoryPool& pool, bool caseInsensitive, const char* patternStr, unsigned patternLen, const char* escapeStr, unsigned escapeLen); + ~SubstringSimilarRegex(); + +private: + static void finalizer(SubstringSimilarRegex* self); public: bool matches(const char* buffer, unsigned bufferLen, unsigned* resultStart, unsigned* resultLength); private: + void* finalizerToken = nullptr; AutoPtr regexp; }; diff --git a/src/common/classes/alloc.cpp b/src/common/classes/alloc.cpp index 26ddc0ce04..ba97e8958b 100644 --- a/src/common/classes/alloc.cpp +++ b/src/common/classes/alloc.cpp @@ -1932,6 +1932,8 @@ private: MemPool* next; MemPool* child; #endif + +friend class MemoryPool; }; @@ -2851,6 +2853,16 @@ void MemoryPool::deallocate(void* block) FB_NOTHROW void MemoryPool::deletePool(MemoryPool* pool) { + while (pool->finalizers) + { + auto next = pool->finalizers->next; + pool->finalizers->next = pool->finalizers->prev = nullptr; + + pool->finalizers->finalizer(pool->finalizers->object); + + pool->finalizers = next; + } + MemPool::deletePool(pool->pool); pool->pool = NULL; delete pool; @@ -2870,6 +2882,57 @@ void MemoryPool::print_contents(const char* filename, unsigned flags, const char #endif } +void* MemoryPool::internalRegisterFinalizer(void (*finalizer)(void*), void* object) +{ + fb_assert(finalizer); + + FinalizerEntry* entry = static_cast(allocate(sizeof(FinalizerEntry) ALLOC_ARGS)); + + MutexLockGuard guard(pool->mutex, "MemoryPool::internalRegisterFinalizer"); + + entry->finalizer = finalizer; + entry->object = object; + entry->prev = nullptr; + entry->next = finalizers; + + if (finalizers) + { + fb_assert(!finalizers->prev); + finalizers->prev = entry; + } + + finalizers = entry; + + return entry; +} + +void MemoryPool::unregisterFinalizer(void*& token) +{ + FinalizerEntry* entry = static_cast(token); + + if (entry->finalizer) + { + MutexLockGuard guard(pool->mutex, "MemoryPool::unregisterFinalizer"); + + if (entry->prev) + { + fb_assert(entry->prev->next == entry); + entry->prev->next = entry->next; + } + else + finalizers = entry->next; + + if (entry->next) + entry->next->prev = entry->prev; + } + else + fb_assert(!entry->next && !entry->prev); + + deallocate(entry); + + token = nullptr; +} + #if defined(DEV_BUILD) void AutoStorage::ProbeStack() const diff --git a/src/common/classes/alloc.h b/src/common/classes/alloc.h index 2c844778bf..b38a2c2f04 100644 --- a/src/common/classes/alloc.h +++ b/src/common/classes/alloc.h @@ -239,6 +239,27 @@ public: // The same routine, but more easily callable from the debugger void print_contents(const char* filename, unsigned flags = 0, const char* filter_path = 0) FB_NOTHROW; + template void* registerFinalizer(void (*finalizer)(T*), T* object) + { + return internalRegisterFinalizer(reinterpret_cast(finalizer), object); + } + + void unregisterFinalizer(void*& token); + +private: + void* internalRegisterFinalizer(void (*finalizer)(void*), void* object); + +private: + struct FinalizerEntry + { + void (*finalizer)(void*); + void* object; + FinalizerEntry* prev; + FinalizerEntry* next; + }; + + FinalizerEntry* finalizers = nullptr; + friend class MemPool; }; From cbf966278698a57f9eb4810258642cf4b658fe67 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Mon, 26 Aug 2019 21:57:48 -0300 Subject: [PATCH 015/274] Remove outdated comments. --- src/common/SimilarToRegex.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/common/SimilarToRegex.h b/src/common/SimilarToRegex.h index 8244eaf1ad..ba276fc02c 100644 --- a/src/common/SimilarToRegex.h +++ b/src/common/SimilarToRegex.h @@ -30,7 +30,6 @@ namespace Firebird { -//// FIXME: Leak re2::RE2 when destroyed by pool. class SimilarToRegex : public PermanentStorage { public: @@ -56,7 +55,6 @@ private: AutoPtr regexp; }; -//// FIXME: Leak re2::RE2 when destroyed by pool. // Given a regular expression R1#R2#R3 and the string S: // - Find the shortest substring of S that matches R1 while the remainder (S23) matches R2R3; // - Find the longest (S2) substring of S23 that matches R2 while the remainder matches R3; From 768aec89376f57572fae1ab0f507cd14200b8343 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Tue, 27 Aug 2019 23:18:31 -0300 Subject: [PATCH 016/274] Avoid cast. --- src/common/SimilarToRegex.cpp | 12 +++--- src/common/SimilarToRegex.h | 8 ++-- src/common/classes/alloc.cpp | 57 ++++++++++++--------------- src/common/classes/alloc.h | 74 ++++++++++++++++++++++++----------- 4 files changed, 88 insertions(+), 63 deletions(-) diff --git a/src/common/SimilarToRegex.cpp b/src/common/SimilarToRegex.cpp index c32f55499d..86fe941531 100644 --- a/src/common/SimilarToRegex.cpp +++ b/src/common/SimilarToRegex.cpp @@ -740,15 +740,15 @@ SimilarToRegex::SimilarToRegex(MemoryPool& pool, bool caseInsensitive, FLAG_GROUP_CAPTURE | FLAG_PREFER_FEWER | (caseInsensitive ? FLAG_CASE_INSENSITIVE : 0), patternStr, patternLen, escapeStr, escapeLen); - finalizerToken = pool.registerFinalizer(finalizer, this); + finalizer = pool.registerFinalizer(finalize, this); } SimilarToRegex::~SimilarToRegex() { - getPool().unregisterFinalizer(finalizerToken); + getPool().unregisterFinalizer(finalizer); } -void SimilarToRegex::finalizer(SimilarToRegex* self) +void SimilarToRegex::finalize(SimilarToRegex* self) { self->regexp.reset(); } @@ -810,15 +810,15 @@ SubstringSimilarRegex::SubstringSimilarRegex(MemoryPool& pool, bool caseInsensit (caseInsensitive ? FLAG_CASE_INSENSITIVE : 0), patternStr, patternLen, escapeStr, escapeLen); - finalizerToken = pool.registerFinalizer(finalizer, this); + finalizer = pool.registerFinalizer(finalize, this); } SubstringSimilarRegex::~SubstringSimilarRegex() { - getPool().unregisterFinalizer(finalizerToken); + getPool().unregisterFinalizer(finalizer); } -void SubstringSimilarRegex::finalizer(SubstringSimilarRegex* self) +void SubstringSimilarRegex::finalize(SubstringSimilarRegex* self) { self->regexp.reset(); } diff --git a/src/common/SimilarToRegex.h b/src/common/SimilarToRegex.h index ba276fc02c..a233435079 100644 --- a/src/common/SimilarToRegex.h +++ b/src/common/SimilarToRegex.h @@ -45,13 +45,13 @@ public: ~SimilarToRegex(); private: - static void finalizer(SimilarToRegex* self); + static void finalize(SimilarToRegex* self); public: bool matches(const char* buffer, unsigned bufferLen, Array* matchPosArray = nullptr); private: - void* finalizerToken = nullptr; + MemoryPool::Finalizer* finalizer = nullptr; AutoPtr regexp; }; @@ -67,13 +67,13 @@ public: ~SubstringSimilarRegex(); private: - static void finalizer(SubstringSimilarRegex* self); + static void finalize(SubstringSimilarRegex* self); public: bool matches(const char* buffer, unsigned bufferLen, unsigned* resultStart, unsigned* resultLength); private: - void* finalizerToken = nullptr; + MemoryPool::Finalizer* finalizer = nullptr; AutoPtr regexp; }; diff --git a/src/common/classes/alloc.cpp b/src/common/classes/alloc.cpp index ba97e8958b..59e41b18e9 100644 --- a/src/common/classes/alloc.cpp +++ b/src/common/classes/alloc.cpp @@ -2855,12 +2855,19 @@ void MemoryPool::deletePool(MemoryPool* pool) { while (pool->finalizers) { - auto next = pool->finalizers->next; - pool->finalizers->next = pool->finalizers->prev = nullptr; + auto finalizer = pool->finalizers; + fb_assert(!finalizer->prev); - pool->finalizers->finalizer(pool->finalizers->object); + pool->finalizers = finalizer->next; - pool->finalizers = next; + if (pool->finalizers) + { + fb_assert(pool->finalizers->prev == finalizer); + pool->finalizers->prev = nullptr; + } + + finalizer->next = nullptr; + finalizer->finalize(); } MemPool::deletePool(pool->pool); @@ -2882,55 +2889,43 @@ void MemoryPool::print_contents(const char* filename, unsigned flags, const char #endif } -void* MemoryPool::internalRegisterFinalizer(void (*finalizer)(void*), void* object) +void MemoryPool::internalRegisterFinalizer(Finalizer* finalizer) { fb_assert(finalizer); - FinalizerEntry* entry = static_cast(allocate(sizeof(FinalizerEntry) ALLOC_ARGS)); - MutexLockGuard guard(pool->mutex, "MemoryPool::internalRegisterFinalizer"); - entry->finalizer = finalizer; - entry->object = object; - entry->prev = nullptr; - entry->next = finalizers; + finalizer->prev = nullptr; + finalizer->next = finalizers; if (finalizers) { fb_assert(!finalizers->prev); - finalizers->prev = entry; + finalizers->prev = finalizer; } - finalizers = entry; - - return entry; + finalizers = finalizer; } -void MemoryPool::unregisterFinalizer(void*& token) +void MemoryPool::unregisterFinalizer(Finalizer*& finalizer) { - FinalizerEntry* entry = static_cast(token); - - if (entry->finalizer) - { + { // scope MutexLockGuard guard(pool->mutex, "MemoryPool::unregisterFinalizer"); - if (entry->prev) + if (finalizer->prev) { - fb_assert(entry->prev->next == entry); - entry->prev->next = entry->next; + fb_assert(finalizer->prev->next == finalizer); + finalizer->prev->next = finalizer->next; } else - finalizers = entry->next; + finalizers = finalizer->next; - if (entry->next) - entry->next->prev = entry->prev; + if (finalizer->next) + finalizer->next->prev = finalizer->prev; } - else - fb_assert(!entry->next && !entry->prev); - deallocate(entry); - - token = nullptr; + delete finalizer; + finalizer = nullptr; } diff --git a/src/common/classes/alloc.h b/src/common/classes/alloc.h index b38a2c2f04..845a87e826 100644 --- a/src/common/classes/alloc.h +++ b/src/common/classes/alloc.h @@ -76,6 +76,16 @@ #define FB_NOTHROW throw() #endif +#ifdef DEBUG_GDS_ALLOC +#define FB_NEW new(__FILE__, __LINE__) +#define FB_NEW_POOL(pool) new(pool, __FILE__, __LINE__) +#define FB_NEW_RPT(pool, count) new(pool, count, __FILE__, __LINE__) +#else // DEBUG_GDS_ALLOC +#define FB_NEW new +#define FB_NEW_POOL(pool) new(pool) +#define FB_NEW_RPT(pool, count) new(pool, count) +#endif // DEBUG_GDS_ALLOC + namespace Firebird { // Alignment for all memory blocks @@ -239,26 +249,53 @@ public: // The same routine, but more easily callable from the debugger void print_contents(const char* filename, unsigned flags = 0, const char* filter_path = 0) FB_NOTHROW; - template void* registerFinalizer(void (*finalizer)(T*), T* object) +public: + struct Finalizer { - return internalRegisterFinalizer(reinterpret_cast(finalizer), object); - } + virtual ~Finalizer() + { + } - void unregisterFinalizer(void*& token); + virtual void finalize() = 0; -private: - void* internalRegisterFinalizer(void (*finalizer)(void*), void* object); - -private: - struct FinalizerEntry - { - void (*finalizer)(void*); - void* object; - FinalizerEntry* prev; - FinalizerEntry* next; + Finalizer* prev = nullptr; + Finalizer* next = nullptr; }; - FinalizerEntry* finalizers = nullptr; + template Finalizer* registerFinalizer(void (*func)(T*), T* object) + { + struct FinalizerImpl : Finalizer + { + FinalizerImpl(void (*aFunc)(T*), T* aObject) + : func(aFunc), + object(aObject) + { + } + + void finalize() override + { + func(object); + } + + void (*func)(T*); + T* object; + }; + + fb_assert(func); + FinalizerImpl* finalizer = FB_NEW_POOL(*this) FinalizerImpl(func, object); + + internalRegisterFinalizer(finalizer); + + return finalizer; + } + + void unregisterFinalizer(Finalizer*& finalizer); + +private: + void internalRegisterFinalizer(Finalizer* entry); + +private: + Finalizer* finalizers = nullptr; friend class MemPool; }; @@ -377,13 +414,6 @@ inline void operator delete[](void* mem) FB_NOTHROW #pragma clang diagnostic pop #endif -#define FB_NEW new(__FILE__, __LINE__) -#define FB_NEW_POOL(pool) new(pool, __FILE__, __LINE__) -#define FB_NEW_RPT(pool, count) new(pool, count, __FILE__, __LINE__) -#else // DEBUG_GDS_ALLOC -#define FB_NEW new -#define FB_NEW_POOL(pool) new(pool) -#define FB_NEW_RPT(pool, count) new(pool, count) #endif // DEBUG_GDS_ALLOC #ifndef USE_SYSTEM_NEW From 09474b099acd1268e1e13e40be851301fd24bb93 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Wed, 28 Aug 2019 13:02:27 -0300 Subject: [PATCH 017/274] Support for NONE/OCTETS. --- src/burp/burp.cpp | 2 +- src/common/SimilarToRegex.cpp | 175 +++++++++++--------- src/common/SimilarToRegex.h | 11 +- src/jrd/Collation.cpp | 86 ++++++---- src/jrd/replication/Manager.cpp | 4 +- src/jrd/validation.cpp | 2 +- src/utilities/ntrace/TraceConfiguration.cpp | 6 +- src/utilities/ntrace/TracePluginImpl.cpp | 4 +- 8 files changed, 174 insertions(+), 116 deletions(-) diff --git a/src/burp/burp.cpp b/src/burp/burp.cpp index 75880da7ed..dc72fb6c0d 100644 --- a/src/burp/burp.cpp +++ b/src/burp/burp.cpp @@ -2544,7 +2544,7 @@ void BurpGlobals::setupSkipData(const Firebird::string& regexp) BurpGlobals* tdgbl = BurpGlobals::getSpecific(); skipDataMatcher.reset(FB_NEW_POOL(tdgbl->getPool()) Firebird::SimilarToRegex( - tdgbl->getPool(), true, + tdgbl->getPool(), Firebird::SimilarToRegex::FLAG_CASE_INSENSITIVE, filter.c_str(), filter.length(), "\\", 1)); } diff --git a/src/common/SimilarToRegex.cpp b/src/common/SimilarToRegex.cpp index 86fe941531..0e11122858 100644 --- a/src/common/SimilarToRegex.cpp +++ b/src/common/SimilarToRegex.cpp @@ -27,11 +27,34 @@ using namespace Firebird; namespace { - static const unsigned FLAG_PREFER_FEWER = 0x01; - static const unsigned FLAG_CASE_INSENSITIVE = 0x02; - static const unsigned FLAG_GROUP_CAPTURE = 0x04; + bool hasChar(int32_t len, int32_t pos) + { + return pos < len; + } + + UChar32 getChar(bool latin, const char* str, int32_t len, int32_t& pos) + { + fb_assert(hasChar(len, pos)); + UChar32 c; + + if (latin) + c = str[pos++]; + else + { + U8_NEXT(str, pos, len, c); + + if (c < 0) + status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); + } + + return c; + } + + static const unsigned COMP_FLAG_PREFER_FEWER = 0x01; + static const unsigned COMP_FLAG_GROUP_CAPTURE = 0x02; + static const unsigned COMP_FLAG_CASE_INSENSITIVE = 0x04; + static const unsigned COMP_FLAG_LATIN = 0x08; - //// TODO: Verify usage of U8_NEXT_UNSAFE. class SimilarToCompiler { public: @@ -48,19 +71,19 @@ namespace if (escapeStr) { int32_t escapePos = 0; - U8_NEXT_UNSAFE(escapeStr, escapePos, escapeChar); + escapeChar = getChar(flags & COMP_FLAG_LATIN, escapeStr, escapeLen, escapePos); if (escapePos != escapeLen) status_exception::raise(Arg::Gds(isc_escape_invalid)); } - if (flags & FLAG_GROUP_CAPTURE) + if (flags & COMP_FLAG_GROUP_CAPTURE) re2PatternStr.append("("); int parseFlags; parseExpr(&parseFlags); - if (flags & FLAG_GROUP_CAPTURE) + if (flags & COMP_FLAG_GROUP_CAPTURE) re2PatternStr.append(")"); // Check for proper termination. @@ -70,7 +93,8 @@ namespace RE2::Options options; options.set_log_errors(false); options.set_dot_nl(true); - options.set_case_sensitive(!(flags & FLAG_CASE_INSENSITIVE)); + options.set_case_sensitive(!(flags & COMP_FLAG_CASE_INSENSITIVE)); + options.set_utf8(!(flags & COMP_FLAG_LATIN)); re2::StringPiece sp((const char*) re2PatternStr.c_str(), re2PatternStr.length()); regexp = FB_NEW_POOL(pool) RE2(sp, options); @@ -79,23 +103,20 @@ namespace status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); } - bool hasChar() + bool hasPatternChar() { return patternPos < patternLen; } - UChar32 getChar() + UChar32 getPatternChar() { - fb_assert(hasChar()); - UChar32 c; - U8_NEXT_UNSAFE(patternStr, patternPos, c); - return c; + return getChar(flags & COMP_FLAG_LATIN, patternStr, patternLen, patternPos); } - UChar32 peekChar() + UChar32 peekPatternChar() { auto savePos = patternPos; - auto c = getChar(); + auto c = getPatternChar(); patternPos = savePos; return c; } @@ -169,7 +190,7 @@ namespace auto savePos = patternPos; UChar32 c; - if (!hasChar() || (c = getChar()) != '|') + if (!hasPatternChar() || (c = getPatternChar()) != '|') { patternPos = savePos; break; @@ -185,9 +206,9 @@ namespace bool first = true; - while (hasChar()) + while (hasPatternChar()) { - auto c = peekChar(); + auto c = peekPatternChar(); if (c != '|' && c != ')') { @@ -214,7 +235,7 @@ namespace UChar32 op; - if (!hasChar() || !isRep((op = peekChar()))) + if (!hasPatternChar() || !isRep((op = peekPatternChar()))) { *parseFlagOut = parseFlags; return; @@ -227,19 +248,19 @@ namespace if (op == '*') { - re2PatternStr.append((flags & FLAG_PREFER_FEWER) ? "*?" : "*"); + re2PatternStr.append((flags & COMP_FLAG_PREFER_FEWER) ? "*?" : "*"); *parseFlagOut = 0; ++patternPos; } else if (op == '+') { - re2PatternStr.append((flags & FLAG_PREFER_FEWER) ? "+?" : "+"); + re2PatternStr.append((flags & COMP_FLAG_PREFER_FEWER) ? "+?" : "+"); *parseFlagOut = PARSE_FLAG_NOT_EMPTY; ++patternPos; } else if (op == '?') { - re2PatternStr.append((flags & FLAG_PREFER_FEWER) ? "??" : "?"); + re2PatternStr.append((flags & COMP_FLAG_PREFER_FEWER) ? "??" : "?"); *parseFlagOut = 0; ++patternPos; } @@ -252,10 +273,10 @@ namespace while (true) { - if (!hasChar()) + if (!hasPatternChar()) status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - UChar32 c = getChar(); + UChar32 c = getPatternChar(); if (c == '}') { @@ -288,11 +309,11 @@ namespace re2PatternStr.append(patternStr + repeatStart, patternStr + patternPos); - if (flags & FLAG_PREFER_FEWER) + if (flags & COMP_FLAG_PREFER_FEWER) re2PatternStr.append("?"); } - if (hasChar() && isRep(peekChar())) + if (hasPatternChar() && isRep(peekPatternChar())) status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); } @@ -300,9 +321,9 @@ namespace { *parseFlagOut = 0; - fb_assert(hasChar()); + fb_assert(hasPatternChar()); auto savePos = patternPos; - auto op = getChar(); + auto op = getPatternChar(); if (op == '_') { @@ -312,7 +333,7 @@ namespace } else if (op == '%') { - re2PatternStr.append((flags & FLAG_PREFER_FEWER) ? ".*?" : ".*"); + re2PatternStr.append((flags & COMP_FLAG_PREFER_FEWER) ? ".*?" : ".*"); return; } else if (op == '[') @@ -344,21 +365,21 @@ namespace do { - if (!hasChar()) + if (!hasPatternChar()) status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); unsigned charSavePos = patternPos; - UChar32 c = getChar(); + UChar32 c = getPatternChar(); bool range = false; bool charClass = false; if (useEscape && c == escapeChar) { - if (!hasChar()) + if (!hasPatternChar()) status_exception::raise(Arg::Gds(isc_escape_invalid)); charSavePos = patternPos; - c = getChar(); + c = getPatternChar(); if (!(c == escapeChar || isSpecial(c))) status_exception::raise(Arg::Gds(isc_escape_invalid)); @@ -384,17 +405,17 @@ namespace if (charClass) { - if (!hasChar() || getChar() != ':') + if (!hasPatternChar() || getPatternChar() != ':') status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); charSavePos = patternPos; - while (hasChar() && getChar() != ':') + while (hasPatternChar() && getPatternChar() != ':') ; const SLONG len = patternPos - charSavePos - 1; - if (!hasChar() || getChar() != ']') + if (!hasPatternChar() || getPatternChar() != ']') status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); for (item.clazz = 0; item.clazz < FB_NELEM(classes); ++item.clazz) @@ -416,20 +437,20 @@ namespace item.firstStart = item.lastStart = charSavePos; item.firstEnd = item.lastEnd = patternPos; - if (hasChar() && peekChar() == '-') + if (hasPatternChar() && peekPatternChar() == '-') { - getChar(); + getPatternChar(); charSavePos = patternPos; - c = getChar(); + c = getPatternChar(); if (useEscape && c == escapeChar) { - if (!hasChar()) + if (!hasPatternChar()) status_exception::raise(Arg::Gds(isc_escape_invalid)); charSavePos = patternPos; - c = getChar(); + c = getPatternChar(); if (!(c == escapeChar || isSpecial(c))) status_exception::raise(Arg::Gds(isc_escape_invalid)); @@ -442,9 +463,9 @@ namespace items.add(item); - if (!hasChar()) + if (!hasPatternChar()) status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - } while (peekChar() != ']'); + } while (peekPatternChar() != ']'); auto appendItem = [&](const Item& item, bool negated) { if (item.clazz != -1) @@ -457,11 +478,10 @@ namespace { if (negated) { - UChar32 c; char hex[20]; int32_t cPos = item.firstStart; - U8_NEXT_UNSAFE(patternStr, cPos, c); + UChar32 c = getChar(flags & COMP_FLAG_LATIN, patternStr, patternLen, cPos); if (c > 0) { @@ -473,7 +493,7 @@ namespace } cPos = item.lastStart; - U8_NEXT_UNSAFE(patternStr, cPos, c); + c = getChar(flags & COMP_FLAG_LATIN, patternStr, patternLen, cPos); if (c < 0x10FFFF) { @@ -537,17 +557,17 @@ namespace re2PatternStr.append("]"); } - getChar(); + getPatternChar(); *parseFlagOut |= PARSE_FLAG_NOT_EMPTY; } else if (op == '(') { - re2PatternStr.append(flags & FLAG_GROUP_CAPTURE ? "(" : "(?:"); + re2PatternStr.append(flags & COMP_FLAG_GROUP_CAPTURE ? "(" : "(?:"); int parseFlags; parseExpr(&parseFlags); - if (!hasChar() || getChar() != ')') + if (!hasPatternChar() || getPatternChar() != ')') status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); re2PatternStr.append(")"); @@ -563,12 +583,12 @@ namespace do { auto charSavePos = patternPos; - op = getChar(); + op = getPatternChar(); if (useEscape && op == escapeChar) { charSavePos = patternPos; - op = getChar(); + op = getPatternChar(); if (!isSpecial(op) && op != escapeChar) status_exception::raise(Arg::Gds(isc_escape_invalid)); @@ -589,7 +609,7 @@ namespace re2PatternStr.append(patternStr + charSavePos, patternStr + patternPos); } - } while (!controlChar && hasChar()); + } while (!controlChar && hasPatternChar()); if (patternPos == savePos) status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); @@ -604,7 +624,7 @@ namespace } private: - static const int PARSE_FLAG_NOT_EMPTY = 1; // known never to match empty string + static const int PARSE_FLAG_NOT_EMPTY = 1; // known never to match empty string string re2PatternStr; const char* patternStr; @@ -618,15 +638,16 @@ namespace class SubstringSimilarCompiler { public: - SubstringSimilarCompiler(MemoryPool& pool, AutoPtr& regexp, unsigned flags, + SubstringSimilarCompiler(MemoryPool& pool, AutoPtr& regexp, unsigned aFlags, const char* aPatternStr, unsigned aPatternLen, const char* escapeStr, unsigned escapeLen) - : patternStr(aPatternStr), + : flags(aFlags), + patternStr(aPatternStr), patternPos(0), patternLen(aPatternLen) { int32_t escapePos = 0; - U8_NEXT_UNSAFE(escapeStr, escapePos, escapeChar); + escapeChar = getChar(flags & COMP_FLAG_LATIN, escapeStr, escapeLen, escapePos); if (escapePos != escapeLen) status_exception::raise(Arg::Gds(isc_escape_invalid)); @@ -634,17 +655,17 @@ namespace unsigned positions[2]; unsigned part = 0; - while (hasChar()) + while (hasPatternChar()) { - auto c = getChar(); + auto c = getPatternChar(); if (c != escapeChar) continue; - if (!hasChar()) + if (!hasPatternChar()) status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); - c = getChar(); + c = getPatternChar(); if (c == '"') { @@ -660,13 +681,13 @@ namespace AutoPtr regexp1, regexp2, regexp3; - SimilarToCompiler compiler1(pool, regexp1, FLAG_PREFER_FEWER, + SimilarToCompiler compiler1(pool, regexp1, COMP_FLAG_PREFER_FEWER, aPatternStr, positions[0] - escapeLen - 1, escapeStr, escapeLen); SimilarToCompiler compiler2(pool, regexp2, 0, aPatternStr + positions[0], positions[1] - positions[0] - escapeLen - 1, escapeStr, escapeLen); - SimilarToCompiler compiler3(pool, regexp3, FLAG_PREFER_FEWER, + SimilarToCompiler compiler3(pool, regexp3, COMP_FLAG_PREFER_FEWER, aPatternStr + positions[1], patternLen - positions[1], escapeStr, escapeLen); string finalRe2Pattern; @@ -691,7 +712,8 @@ namespace RE2::Options options; options.set_log_errors(false); options.set_dot_nl(true); - options.set_case_sensitive(!(flags & FLAG_CASE_INSENSITIVE)); + options.set_case_sensitive(!(flags & COMP_FLAG_CASE_INSENSITIVE)); + options.set_utf8(!(flags & COMP_FLAG_LATIN)); re2::StringPiece sp((const char*) finalRe2Pattern.c_str(), finalRe2Pattern.length()); regexp = FB_NEW_POOL(pool) RE2(sp, options); @@ -700,28 +722,26 @@ namespace status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); } - bool hasChar() + bool hasPatternChar() { return patternPos < patternLen; } - UChar32 getChar() + UChar32 getPatternChar() { - fb_assert(hasChar()); - UChar32 c; - U8_NEXT_UNSAFE(patternStr, patternPos, c); - return c; + return getChar(flags & COMP_FLAG_LATIN, patternStr, patternLen, patternPos); } - UChar32 peekChar() + UChar32 peekPatternChar() { auto savePos = patternPos; - auto c = getChar(); + auto c = getPatternChar(); patternPos = savePos; return c; } private: + unsigned flags; const char* patternStr; int32_t patternPos; int32_t patternLen; @@ -732,12 +752,14 @@ namespace namespace Firebird { -SimilarToRegex::SimilarToRegex(MemoryPool& pool, bool caseInsensitive, +SimilarToRegex::SimilarToRegex(MemoryPool& pool, unsigned flags, const char* patternStr, unsigned patternLen, const char* escapeStr, unsigned escapeLen) : PermanentStorage(pool) { SimilarToCompiler compiler(pool, regexp, - FLAG_GROUP_CAPTURE | FLAG_PREFER_FEWER | (caseInsensitive ? FLAG_CASE_INSENSITIVE : 0), + COMP_FLAG_GROUP_CAPTURE | COMP_FLAG_PREFER_FEWER | + ((flags & FLAG_CASE_INSENSITIVE) ? COMP_FLAG_CASE_INSENSITIVE : 0) | + ((flags & FLAG_LATIN) ? COMP_FLAG_LATIN : 0), patternStr, patternLen, escapeStr, escapeLen); finalizer = pool.registerFinalizer(finalize, this); @@ -802,12 +824,13 @@ bool SimilarToRegex::matches(const char* buffer, unsigned bufferLen, ArraygetCharSet()->getId()); - UCharBuffer patternBuffer, escapeBuffer; - converter.convert(patternLen, patternStr, patternBuffer); + const auto charSetId = textType->getCharSet()->getId(); + unsigned flags = 0; - if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) - UnicodeUtil::utf8Normalize(patternBuffer); - - if (escapeStr) + if (charSetId != CS_NONE && charSetId != CS_BINARY) { - converter.convert(escapeLen, escapeStr, escapeBuffer); + flags |= (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ? + SimilarToRegex::FLAG_CASE_INSENSITIVE : 0; + + CsConvert converter = INTL_convert_lookup(tdbb, CS_UTF8, charSetId); + + converter.convert(patternLen, patternStr, patternBuffer); if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) - UnicodeUtil::utf8Normalize(escapeBuffer); - } + UnicodeUtil::utf8Normalize(patternBuffer); - regex = FB_NEW_POOL(pool) SimilarToRegex(pool, - (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE), - (const char*) patternBuffer.begin(), patternBuffer.getCount(), - (escapeStr ? (const char*) escapeBuffer.begin() : nullptr), escapeBuffer.getCount()); + patternStr = patternBuffer.begin(); + patternLen = patternBuffer.getCount(); + + if (escapeStr) + { + converter.convert(escapeLen, escapeStr, escapeBuffer); + + if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + UnicodeUtil::utf8Normalize(escapeBuffer); + + escapeStr = escapeBuffer.begin(); + escapeLen = escapeBuffer.getCount(); + } + } + else + flags |= SimilarToRegex::FLAG_LATIN; + + regex = FB_NEW_POOL(pool) SimilarToRegex(pool, flags, + (const char*) patternStr, patternLen, + (escapeStr ? (const char*) escapeStr : nullptr), escapeLen); } public: @@ -193,27 +208,40 @@ public: resultStart(0), resultLength(0) { - CsConvert converter = INTL_convert_lookup(tdbb, textType->getCharSet()->getId(), CS_UTF8); - UCharBuffer patternBuffer, escapeBuffer; - converter.convert(patternLen, patternStr, patternBuffer); + const auto charSetId = textType->getCharSet()->getId(); + unsigned flags = 0; - if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) - UnicodeUtil::utf8Normalize(patternBuffer); - - if (escapeStr) + if (charSetId != CS_NONE && charSetId != CS_BINARY) { - converter.convert(escapeLen, escapeStr, escapeBuffer); + CsConvert converter = INTL_convert_lookup(tdbb, textType->getCharSet()->getId(), CS_UTF8); + + converter.convert(patternLen, patternStr, patternBuffer); if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) - UnicodeUtil::utf8Normalize(escapeBuffer); - } + UnicodeUtil::utf8Normalize(patternBuffer); - regex = FB_NEW_POOL(pool) SubstringSimilarRegex(pool, - (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE), - (const char*) patternBuffer.begin(), patternBuffer.getCount(), - (escapeStr ? (const char*) escapeBuffer.begin() : nullptr), escapeBuffer.getCount()); + patternStr = patternBuffer.begin(); + patternLen = patternBuffer.getCount(); + + if (escapeStr) + { + converter.convert(escapeLen, escapeStr, escapeBuffer); + + if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + UnicodeUtil::utf8Normalize(escapeBuffer); + + escapeStr = escapeBuffer.begin(); + escapeLen = escapeBuffer.getCount(); + } + } + else + flags |= SimilarToRegex::FLAG_LATIN; + + regex = FB_NEW_POOL(pool) SubstringSimilarRegex(pool, flags, + (const char*) patternStr, patternLen, + (escapeStr ? (const char*) escapeStr : nullptr), escapeLen); } virtual ~Re2SubstringSimilarMatcher() diff --git a/src/jrd/replication/Manager.cpp b/src/jrd/replication/Manager.cpp index a8960c8328..6a232329dd 100644 --- a/src/jrd/replication/Manager.cpp +++ b/src/jrd/replication/Manager.cpp @@ -53,7 +53,7 @@ TableMatcher::TableMatcher(MemoryPool& pool, if (includeFilter.hasData()) { m_includeMatcher.reset(FB_NEW_POOL(pool) SimilarToRegex( - pool, true, + pool, SimilarToRegex::FLAG_CASE_INSENSITIVE, includeFilter.c_str(), includeFilter.length(), "\\", 1)); } @@ -61,7 +61,7 @@ TableMatcher::TableMatcher(MemoryPool& pool, if (excludeFilter.hasData()) { m_excludeMatcher.reset(FB_NEW_POOL(pool) SimilarToRegex( - pool, true, + pool, SimilarToRegex::FLAG_CASE_INSENSITIVE, excludeFilter.c_str(), excludeFilter.length(), "\\", 1)); } diff --git a/src/jrd/validation.cpp b/src/jrd/validation.cpp index 409f824d0e..129eab53d8 100644 --- a/src/jrd/validation.cpp +++ b/src/jrd/validation.cpp @@ -603,7 +603,7 @@ static SimilarToRegex* createPatternMatcher(thread_db* tdbb, const char* pattern //// TODO: Should this be different than trace and replication //// and use case sensitive matcher? matcher = FB_NEW_POOL(*tdbb->getDefaultPool()) SimilarToRegex( - *tdbb->getDefaultPool(), false, + *tdbb->getDefaultPool(), 0, pattern, len, "\\", 1); } diff --git a/src/utilities/ntrace/TraceConfiguration.cpp b/src/utilities/ntrace/TraceConfiguration.cpp index 2c4136e1f3..e28d5a5c46 100644 --- a/src/utilities/ntrace/TraceConfiguration.cpp +++ b/src/utilities/ntrace/TraceConfiguration.cpp @@ -134,14 +134,14 @@ void TraceCfgReader::readConfig() try { #ifdef WIN_NT // !CASE_SENSITIVITY - const bool caseInsensitive = true; + const unsigned regexFlags = SimilarToRegex::FLAG_CASE_INSENSITIVE; #else - const bool caseInsensitive = false; + const unsigned regexFlags = 0; #endif string utf8Pattern = pattern; ISC_systemToUtf8(utf8Pattern); - SimilarToRegex matcher(*getDefaultMemoryPool(), caseInsensitive, + SimilarToRegex matcher(*getDefaultMemoryPool(), regexFlags, utf8Pattern.c_str(), utf8Pattern.length(), "\\", 1); regExpOk = true; diff --git a/src/utilities/ntrace/TracePluginImpl.cpp b/src/utilities/ntrace/TracePluginImpl.cpp index 214388638c..5f411df60f 100644 --- a/src/utilities/ntrace/TracePluginImpl.cpp +++ b/src/utilities/ntrace/TracePluginImpl.cpp @@ -134,7 +134,7 @@ TracePluginImpl::TracePluginImpl(IPluginBase* plugin, ISC_systemToUtf8(filter); include_matcher = FB_NEW SimilarToRegex( - *getDefaultMemoryPool(), true, + *getDefaultMemoryPool(), SimilarToRegex::FLAG_CASE_INSENSITIVE, filter.c_str(), filter.length(), "\\", 1); } @@ -146,7 +146,7 @@ TracePluginImpl::TracePluginImpl(IPluginBase* plugin, ISC_systemToUtf8(filter); exclude_matcher = FB_NEW SimilarToRegex( - *getDefaultMemoryPool(), true, + *getDefaultMemoryPool(), SimilarToRegex::FLAG_CASE_INSENSITIVE, filter.c_str(), filter.length(), "\\", 1); } From 3aa372f54617bfabd6450511bff2fa4b14125e87 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Mon, 2 Sep 2019 12:53:07 -0300 Subject: [PATCH 018/274] 1) Do not use U8_NEXT as some ICU versions calls a function; 2) Some corrections. --- src/common/SimilarToRegex.cpp | 22 +++++++++++++++++++--- src/common/SimilarToRegex.h | 2 ++ src/jrd/Collation.cpp | 11 ++++++++++- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/common/SimilarToRegex.cpp b/src/common/SimilarToRegex.cpp index 0e11122858..bb4e8f9565 100644 --- a/src/common/SimilarToRegex.cpp +++ b/src/common/SimilarToRegex.cpp @@ -21,6 +21,7 @@ #include "firebird.h" #include "../common/SimilarToRegex.h" #include "../common/StatusArg.h" +#include "../common/unicode_util.h" #include using namespace Firebird; @@ -41,7 +42,7 @@ namespace c = str[pos++]; else { - U8_NEXT(str, pos, len, c); + U8_NEXT_UNSAFE(str, pos, c); if (c < 0) status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); @@ -54,6 +55,7 @@ namespace static const unsigned COMP_FLAG_GROUP_CAPTURE = 0x02; static const unsigned COMP_FLAG_CASE_INSENSITIVE = 0x04; static const unsigned COMP_FLAG_LATIN = 0x08; + static const unsigned COMP_FLAG_WELLFORMED = 0x10; class SimilarToCompiler { @@ -68,8 +70,20 @@ namespace flags(aFlags), useEscape(escapeStr != nullptr) { + if (!(flags & COMP_FLAG_LATIN) && !(flags & COMP_FLAG_WELLFORMED)) + { + if (!Jrd::UnicodeUtil::utf8WellFormed(patternLen, reinterpret_cast(patternStr), nullptr)) + status_exception::raise(Arg::Gds(isc_malformed_string)); + } + if (escapeStr) { + if (!(flags & COMP_FLAG_LATIN) && !(flags & COMP_FLAG_WELLFORMED)) + { + if (!Jrd::UnicodeUtil::utf8WellFormed(escapeLen, reinterpret_cast(escapeStr), nullptr)) + status_exception::raise(Arg::Gds(isc_malformed_string)); + } + int32_t escapePos = 0; escapeChar = getChar(flags & COMP_FLAG_LATIN, escapeStr, escapeLen, escapePos); @@ -759,7 +773,8 @@ SimilarToRegex::SimilarToRegex(MemoryPool& pool, unsigned flags, SimilarToCompiler compiler(pool, regexp, COMP_FLAG_GROUP_CAPTURE | COMP_FLAG_PREFER_FEWER | ((flags & FLAG_CASE_INSENSITIVE) ? COMP_FLAG_CASE_INSENSITIVE : 0) | - ((flags & FLAG_LATIN) ? COMP_FLAG_LATIN : 0), + ((flags & FLAG_LATIN) ? COMP_FLAG_LATIN : 0) | + ((flags & FLAG_WELLFORMED) ? COMP_FLAG_WELLFORMED : 0), patternStr, patternLen, escapeStr, escapeLen); finalizer = pool.registerFinalizer(finalize, this); @@ -830,7 +845,8 @@ SubstringSimilarRegex::SubstringSimilarRegex(MemoryPool& pool, unsigned flags, { SubstringSimilarCompiler compiler(pool, regexp, ((flags & FLAG_CASE_INSENSITIVE) ? COMP_FLAG_CASE_INSENSITIVE : 0) | - ((flags & FLAG_LATIN) ? COMP_FLAG_LATIN : 0), + ((flags & FLAG_LATIN) ? COMP_FLAG_LATIN : 0) | + ((flags & FLAG_WELLFORMED) ? COMP_FLAG_WELLFORMED : 0), patternStr, patternLen, escapeStr, escapeLen); finalizer = pool.registerFinalizer(finalize, this); diff --git a/src/common/SimilarToRegex.h b/src/common/SimilarToRegex.h index a80b99341d..2834f82c99 100644 --- a/src/common/SimilarToRegex.h +++ b/src/common/SimilarToRegex.h @@ -35,6 +35,7 @@ class SimilarToRegex : public PermanentStorage public: static const unsigned FLAG_CASE_INSENSITIVE = 0x1; static const unsigned FLAG_LATIN = 0x2; + static const unsigned FLAG_WELLFORMED = 0x3; struct MatchPos { @@ -67,6 +68,7 @@ class SubstringSimilarRegex : public PermanentStorage public: static const unsigned FLAG_CASE_INSENSITIVE = SimilarToRegex::FLAG_CASE_INSENSITIVE; static const unsigned FLAG_LATIN = SimilarToRegex::FLAG_LATIN; + static const unsigned FLAG_WELLFORMED = SimilarToRegex::FLAG_WELLFORMED; public: SubstringSimilarRegex(MemoryPool& pool, unsigned flags, diff --git a/src/jrd/Collation.cpp b/src/jrd/Collation.cpp index b2be1e6b56..64d05002d9 100644 --- a/src/jrd/Collation.cpp +++ b/src/jrd/Collation.cpp @@ -125,6 +125,9 @@ public: if (charSetId != CS_NONE && charSetId != CS_BINARY) { + if (charSetId != CS_UTF8) + flags |= SimilarToRegex::FLAG_WELLFORMED; + flags |= (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ? SimilarToRegex::FLAG_CASE_INSENSITIVE : 0; @@ -215,6 +218,12 @@ public: if (charSetId != CS_NONE && charSetId != CS_BINARY) { + if (charSetId != CS_UTF8) + flags |= SubstringSimilarRegex::FLAG_WELLFORMED; + + flags |= (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ? + SubstringSimilarRegex::FLAG_CASE_INSENSITIVE : 0; + CsConvert converter = INTL_convert_lookup(tdbb, textType->getCharSet()->getId(), CS_UTF8); converter.convert(patternLen, patternStr, patternBuffer); @@ -237,7 +246,7 @@ public: } } else - flags |= SimilarToRegex::FLAG_LATIN; + flags |= SubstringSimilarRegex::FLAG_LATIN; regex = FB_NEW_POOL(pool) SubstringSimilarRegex(pool, flags, (const char*) patternStr, patternLen, From 76088c21ccad734a87c2465190f9381d35d68341 Mon Sep 17 00:00:00 2001 From: Alex Peshkoff Date: Thu, 5 Sep 2019 19:23:38 +0300 Subject: [PATCH 019/274] Implemented CORE-6048: Provide ability to see current state of DB encryption --- doc/README.monitoring_tables | 7 ++++++- src/include/gen/ids.h | 1 + src/jrd/Monitoring.cpp | 3 +++ src/jrd/fields.h | 2 ++ src/jrd/names.h | 2 ++ src/jrd/relations.h | 1 + 6 files changed, 15 insertions(+), 1 deletion(-) diff --git a/doc/README.monitoring_tables b/doc/README.monitoring_tables index 8fb02654dc..1a5b47edb2 100644 --- a/doc/README.monitoring_tables +++ b/doc/README.monitoring_tables @@ -75,7 +75,12 @@ Monitoring tables 0: normal 1: stalled 2: merge - - MON$CRYPT_PAGE (number of page being encrypted) + - MON$CRYPT_STATE (current encryption state) + 0: not encrypted + 1: encrypted + 2: decrypt in process + 3: encrypt in process + - MON$CRYPT_PAGE (number of page being encrypted / decrypted) - MON$OWNER (database owner name) - MON$SEC_DATABASE (security database) diff --git a/src/include/gen/ids.h b/src/include/gen/ids.h index b6f080ed06..9c3beca6b8 100644 --- a/src/include/gen/ids.h +++ b/src/include/gen/ids.h @@ -494,6 +494,7 @@ const USHORT f_mon_db_crypt_page = 19; const USHORT f_mon_db_owner = 20; const USHORT f_mon_db_secdb = 21; + const USHORT f_mon_db_crypt_state = 22; // Relation 34 (MON$ATTACHMENTS) diff --git a/src/jrd/Monitoring.cpp b/src/jrd/Monitoring.cpp index cf4003a696..f20f9f271a 100644 --- a/src/jrd/Monitoring.cpp +++ b/src/jrd/Monitoring.cpp @@ -829,7 +829,10 @@ void Monitoring::putDatabase(thread_db* tdbb, SnapshotData::DumpRecord& record) // crypt thread status if (database->dbb_crypto_manager) + { record.storeInteger(f_mon_db_crypt_page, database->dbb_crypto_manager->getCurrentPage()); + record.storeInteger(f_mon_db_crypt_state, database->dbb_crypto_manager->getCurrentState()); + } // database owner record.storeString(f_mon_db_owner, database->dbb_owner); diff --git a/src/jrd/fields.h b/src/jrd/fields.h index 3f3739de49..36f75f9497 100644 --- a/src/jrd/fields.h +++ b/src/jrd/fields.h @@ -207,3 +207,5 @@ FIELD(fld_timestamp_tz , nam_timestamp_tz , dtype_timestamp_tz, TIMESTAMP_TZ_SIZE , 0 , NULL , true) FIELD(fld_tz_db_version , nam_tz_db_version , dtype_varying , 10 , dsc_text_type_ascii , NULL , true) + + FIELD(fld_crypt_state , nam_crypt_state , dtype_short , sizeof(SSHORT) , 0 , NULL , true) \ No newline at end of file diff --git a/src/jrd/names.h b/src/jrd/names.h index 7a01f37789..05b405cd47 100644 --- a/src/jrd/names.h +++ b/src/jrd/names.h @@ -228,6 +228,7 @@ NAME("RDB$ATTACHMENT_ID", nam_att_id) NAME("RDB$BACKUP_STATE", nam_backup_state) NAME("RDB$CALL_ID", nam_call_id) NAME("RDB$COUNTER", nam_counter) +NAME("RDB$CRYPT_STATE", nam_crypt_state) NAME("RDB$ISOLATION_MODE", nam_iso_mode) NAME("RDB$LOCK_TIMEOUT", nam_lock_timeout) NAME("RDB$ODS_NUMBER", nam_ods_number) @@ -295,6 +296,7 @@ NAME("MON$CLIENT_VERSION", nam_mon_client_ver) NAME("MON$CONTEXT_VARIABLES", nam_mon_ctx_vars) NAME("MON$CREATION_DATE", nam_mon_created) NAME("MON$CRYPT_PAGE", nam_mon_crypt_page) +NAME("MON$CRYPT_STATE", nam_mon_crypt_state) NAME("MON$DATABASE", nam_mon_database) NAME("MON$DATABASE_NAME", nam_mon_db_name) NAME("MON$EXPLAINED_PLAN", nam_mon_expl_plan) diff --git a/src/jrd/relations.h b/src/jrd/relations.h index 9ad1bd9d18..37ce864eb8 100644 --- a/src/jrd/relations.h +++ b/src/jrd/relations.h @@ -493,6 +493,7 @@ RELATION(nam_mon_database, rel_mon_database, ODS_11_1, rel_virtual) FIELD(f_mon_db_crypt_page, nam_mon_crypt_page, fld_counter, 0, ODS_12_0) FIELD(f_mon_db_owner, nam_mon_owner, fld_user, 0, ODS_12_0) FIELD(f_mon_db_secdb, nam_mon_secdb, fld_sec_db, 0, ODS_12_0) + FIELD(f_mon_db_crypt_state, nam_mon_crypt_state, fld_crypt_state, 0, ODS_13_0) END_RELATION // Relation 34 (MON$ATTACHMENTS) From 6bada8e683ade1481ccdb9afdd86ea26dca5f09f Mon Sep 17 00:00:00 2001 From: hvlad Date: Thu, 5 Sep 2019 22:14:14 +0300 Subject: [PATCH 020/274] Fixed bug CORE-6137 : Server crashes when it run SQL --- src/jrd/recsrc/IndexTableScan.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/jrd/recsrc/IndexTableScan.cpp b/src/jrd/recsrc/IndexTableScan.cpp index 979692dac8..00c3a4719b 100644 --- a/src/jrd/recsrc/IndexTableScan.cpp +++ b/src/jrd/recsrc/IndexTableScan.cpp @@ -49,7 +49,10 @@ IndexTableScan::IndexTableScan(CompilerScratch* csb, const string& alias, { fb_assert(m_index); - FB_SIZE_T size = sizeof(Impure) + 2u * m_length; + // Reserve one excess byte for the upper key - in case when length of + // upper key at retrieval is greater than declared index key length. + // See also comments at openStream(). + FB_SIZE_T size = sizeof(Impure) + 2u * m_length + 1u; size = FB_ALIGN(size, FB_ALIGNMENT); m_offset = size; size += sizeof(index_desc); @@ -507,8 +510,11 @@ UCHAR* IndexTableScan::openStream(thread_db* tdbb, Impure* impure, win* window) temporary_key* limit_ptr = NULL; if (retrieval->irb_upper_count) { - impure->irsb_nav_upper_length = upper.key_length; - memcpy(impure->irsb_nav_data + m_length, upper.key_data, upper.key_length); + // If upper key length is greater than declared key length, we need + // one "excess" byte for correct comparison. Without it there could + // be false equality hits. + impure->irsb_nav_upper_length = MIN(m_length + 1, upper.key_length); + memcpy(impure->irsb_nav_data + m_length, upper.key_data, impure->irsb_nav_upper_length); } if (retrieval->irb_lower_count) @@ -581,6 +587,7 @@ void IndexTableScan::setPosition(thread_db* tdbb, impure->irsb_nav_number = rpb->rpb_number; // save the current key value + fb_assert(key.key_length <= m_length); impure->irsb_nav_length = key.key_length; memcpy(impure->irsb_nav_data, key.key_data, key.key_length); From f012089937dd21523742ce22dc1fb8cd1c50d761 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Fri, 6 Sep 2019 00:03:55 +0000 Subject: [PATCH 021/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index bab0c0a042..18ffb94ba8 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1573 + FORMAL BUILD NUMBER:1575 */ -#define PRODUCT_VER_STRING "4.0.0.1573" -#define FILE_VER_STRING "WI-T4.0.0.1573" -#define LICENSE_VER_STRING "WI-T4.0.0.1573" -#define FILE_VER_NUMBER 4, 0, 0, 1573 +#define PRODUCT_VER_STRING "4.0.0.1575" +#define FILE_VER_STRING "WI-T4.0.0.1575" +#define LICENSE_VER_STRING "WI-T4.0.0.1575" +#define FILE_VER_NUMBER 4, 0, 0, 1575 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1573" +#define FB_BUILD_NO "1575" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 9d83a9f9a9..5135f81cfc 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1573 +BuildNum=1575 NowAt=`pwd` cd `dirname $0` From d5bd3c1524d83f0491c57d0fcc6851de2d1cefd7 Mon Sep 17 00:00:00 2001 From: Alex Peshkoff Date: Fri, 6 Sep 2019 11:53:28 +0300 Subject: [PATCH 022/274] Fixed syntax in doc --- doc/README.monitoring_tables | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/README.monitoring_tables b/doc/README.monitoring_tables index 1a5b47edb2..b6918f661d 100644 --- a/doc/README.monitoring_tables +++ b/doc/README.monitoring_tables @@ -78,8 +78,8 @@ Monitoring tables - MON$CRYPT_STATE (current encryption state) 0: not encrypted 1: encrypted - 2: decrypt in process - 3: encrypt in process + 2: decrypt in progress + 3: encrypt in progress - MON$CRYPT_PAGE (number of page being encrypted / decrypted) - MON$OWNER (database owner name) - MON$SEC_DATABASE (security database) From ee4c8f2f09e5a1aa4fb5f6bbfa4bfc5e63b9b89b Mon Sep 17 00:00:00 2001 From: Alex Peshkoff Date: Fri, 6 Sep 2019 12:49:27 +0300 Subject: [PATCH 023/274] Postfix for CORE-6048 --- src/jrd/types.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/jrd/types.h b/src/jrd/types.h index ae0b86e5ea..cfe38b0235 100644 --- a/src/jrd/types.h +++ b/src/jrd/types.h @@ -196,3 +196,8 @@ TYPE("ROLE", 1, nam_map_to_type) #define SYSTEM_PRIVILEGE(p) TYPE(STRINGIZE(p), int(Jrd::p), nam_system_privileges) #include "SystemPrivileges.h" #undef SYSTEM_PRIVILEGE + +TYPE("NOT ENCRYPTED", 0, nam_mon_crypt_state) +TYPE("ENCRYPTED", 1, nam_mon_crypt_state) +TYPE("DECRYPT IN PROGRESS", 2, nam_mon_crypt_state) +TYPE("ENCRYPT IN PROGRESS", 3, nam_mon_crypt_state) From e5866c902b7f03bf37e913195dd4fa82068101d4 Mon Sep 17 00:00:00 2001 From: Alex Peshkoff Date: Fri, 6 Sep 2019 12:52:19 +0300 Subject: [PATCH 024/274] Frontported fix for CORE-6134 --- src/auth/AuthDbg.cpp | 6 ++++++ src/auth/trusted/AuthSspi.cpp | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/auth/AuthDbg.cpp b/src/auth/AuthDbg.cpp index 3b27994add..2a7a40dd12 100644 --- a/src/auth/AuthDbg.cpp +++ b/src/auth/AuthDbg.cpp @@ -145,6 +145,12 @@ int DebugClient::authenticate(Firebird::CheckStatusWrapper* status, Firebird::IC { try { + if (cb->getLogin()) + { + // user specified login - we should not continue with trusted-like auth + return AUTH_CONTINUE; + } + if (str != "HAND") { str = "HAND"; diff --git a/src/auth/trusted/AuthSspi.cpp b/src/auth/trusted/AuthSspi.cpp index a87702ad88..13f1994789 100644 --- a/src/auth/trusted/AuthSspi.cpp +++ b/src/auth/trusted/AuthSspi.cpp @@ -480,6 +480,12 @@ int WinSspiClient::authenticate(Firebird::CheckStatusWrapper* status, { try { + if (cBlock->getLogin()) + { + // user specified login - we should not continue with trusted-like auth + return AUTH_CONTINUE; + } + sspiData.clear(); unsigned int length; const unsigned char* bytes = cBlock->getData(&length); From 268bc4faf4063e2ab19d737f7644560928c4c75d Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 7 Sep 2019 00:03:53 +0000 Subject: [PATCH 025/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 18ffb94ba8..3c66e14088 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1575 + FORMAL BUILD NUMBER:1597 */ -#define PRODUCT_VER_STRING "4.0.0.1575" -#define FILE_VER_STRING "WI-T4.0.0.1575" -#define LICENSE_VER_STRING "WI-T4.0.0.1575" -#define FILE_VER_NUMBER 4, 0, 0, 1575 +#define PRODUCT_VER_STRING "4.0.0.1597" +#define FILE_VER_STRING "WI-T4.0.0.1597" +#define LICENSE_VER_STRING "WI-T4.0.0.1597" +#define FILE_VER_NUMBER 4, 0, 0, 1597 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1575" +#define FB_BUILD_NO "1597" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 5135f81cfc..856dfe66c2 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1575 +BuildNum=1597 NowAt=`pwd` cd `dirname $0` From e98188a96e09ec717547eb28b3752ac11c4f4e0f Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sat, 7 Sep 2019 10:05:25 -0300 Subject: [PATCH 026/274] Hope to fix VS 2015 build. --- builds/win32/msvc14/FirebirdCommon.props | 2 +- builds/win32/msvc14/common.vcxproj | 10 ++++++---- builds/win32/msvc14/common.vcxproj.filters | 6 ++++++ builds/win32/msvc14/engine.vcxproj | 1 - builds/win32/msvc14/engine.vcxproj.filters | 3 --- builds/win32/msvc14/fbtrace.vcxproj | 2 -- builds/win32/msvc14/fbtrace.vcxproj.filters | 6 ------ 7 files changed, 13 insertions(+), 17 deletions(-) diff --git a/builds/win32/msvc14/FirebirdCommon.props b/builds/win32/msvc14/FirebirdCommon.props index 0611d1d767..c296233d68 100644 --- a/builds/win32/msvc14/FirebirdCommon.props +++ b/builds/win32/msvc14/FirebirdCommon.props @@ -10,7 +10,7 @@ /EHsc- %(AdditionalOptions) true - ../../../src/include;../../../src/include/gen;../../../extern/libtomcrypt/src/headers;../../../extern/libtommath;../../../extern/icu/include;../../../extern/zlib;%(AdditionalIncludeDirectories) + ../../../src/include;../../../src/include/gen;../../../extern/libtomcrypt/src/headers;../../../extern/libtommath;../../../extern/icu/include;../../../extern/zlib;../../../extern/re2;%(AdditionalIncludeDirectories) false diff --git a/builds/win32/msvc14/common.vcxproj b/builds/win32/msvc14/common.vcxproj index 5ea9f623aa..77823eb058 100644 --- a/builds/win32/msvc14/common.vcxproj +++ b/builds/win32/msvc14/common.vcxproj @@ -87,6 +87,7 @@ + @@ -198,6 +199,7 @@ + @@ -300,7 +302,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) @@ -316,7 +318,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) @@ -335,7 +337,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) @@ -353,7 +355,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc14/common.vcxproj.filters b/builds/win32/msvc14/common.vcxproj.filters index 3d10a970b2..a0b2d29362 100644 --- a/builds/win32/msvc14/common.vcxproj.filters +++ b/builds/win32/msvc14/common.vcxproj.filters @@ -228,6 +228,9 @@ common + + common + common @@ -566,6 +569,9 @@ headers + + headers + headers diff --git a/builds/win32/msvc14/engine.vcxproj b/builds/win32/msvc14/engine.vcxproj index 191be3e254..178b2c1ab3 100644 --- a/builds/win32/msvc14/engine.vcxproj +++ b/builds/win32/msvc14/engine.vcxproj @@ -317,7 +317,6 @@ - diff --git a/builds/win32/msvc14/engine.vcxproj.filters b/builds/win32/msvc14/engine.vcxproj.filters index bb75ef49ec..8eac730a06 100644 --- a/builds/win32/msvc14/engine.vcxproj.filters +++ b/builds/win32/msvc14/engine.vcxproj.filters @@ -938,9 +938,6 @@ Header files - - Header files - Header files diff --git a/builds/win32/msvc14/fbtrace.vcxproj b/builds/win32/msvc14/fbtrace.vcxproj index 11df59d835..d311a2f776 100644 --- a/builds/win32/msvc14/fbtrace.vcxproj +++ b/builds/win32/msvc14/fbtrace.vcxproj @@ -182,7 +182,6 @@ - @@ -190,7 +189,6 @@ - diff --git a/builds/win32/msvc14/fbtrace.vcxproj.filters b/builds/win32/msvc14/fbtrace.vcxproj.filters index da6c776679..dc253e7e40 100644 --- a/builds/win32/msvc14/fbtrace.vcxproj.filters +++ b/builds/win32/msvc14/fbtrace.vcxproj.filters @@ -30,9 +30,6 @@ Source Files - - Source Files - @@ -50,9 +47,6 @@ Header Files - - Header Files - From 6e73b0bd4914bb7e6b5135c92bd359e4f515b375 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sun, 8 Sep 2019 00:03:25 +0000 Subject: [PATCH 027/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 3c66e14088..de608c2bde 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1597 + FORMAL BUILD NUMBER:1598 */ -#define PRODUCT_VER_STRING "4.0.0.1597" -#define FILE_VER_STRING "WI-T4.0.0.1597" -#define LICENSE_VER_STRING "WI-T4.0.0.1597" -#define FILE_VER_NUMBER 4, 0, 0, 1597 +#define PRODUCT_VER_STRING "4.0.0.1598" +#define FILE_VER_STRING "WI-T4.0.0.1598" +#define LICENSE_VER_STRING "WI-T4.0.0.1598" +#define FILE_VER_NUMBER 4, 0, 0, 1598 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1597" +#define FB_BUILD_NO "1598" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 856dfe66c2..ea97da7e53 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1597 +BuildNum=1598 NowAt=`pwd` cd `dirname $0` From 38f5d321798a321d5075dc16b09f45a337e4e016 Mon Sep 17 00:00:00 2001 From: Mark Rotteveel Date: Thu, 12 Sep 2019 10:51:50 +0200 Subject: [PATCH 028/274] CORE-5976 increase minimum for multi-database restore --- src/burp/restore.epp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/burp/restore.epp b/src/burp/restore.epp index 7870ee7a0f..64c9505691 100644 --- a/src/burp/restore.epp +++ b/src/burp/restore.epp @@ -653,7 +653,7 @@ void add_files(BurpGlobals* tdgbl, const char* file_name) // store the RDB$FILES records - FB_UINT64 start = FB_CONST64(201); // Magic number, can be taken from some constant? + FB_UINT64 start = FB_CONST64(256); // Magic number, can be taken from some constant? SLONG count = 0; const char* prev_file_name = NULL; From 0a22aa2070b3e31527b8fed4f1941bd470c16008 Mon Sep 17 00:00:00 2001 From: hvlad Date: Fri, 13 Sep 2019 12:24:18 +0300 Subject: [PATCH 029/274] Port forward fix for bug CORE-6142 : Error "connection lost to database" could happen when application creates few local attachments (using XNET) simultaneously --- src/remote/os/win32/xnet.cpp | 41 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/remote/os/win32/xnet.cpp b/src/remote/os/win32/xnet.cpp index 2a45d326d4..2c60807df1 100644 --- a/src/remote/os/win32/xnet.cpp +++ b/src/remote/os/win32/xnet.cpp @@ -534,10 +534,7 @@ bool XnetClientEndPoint::connect_init() if (!xnet_connect_mutex) { if (ERRNO == ERROR_FILE_NOT_FOUND) - { - make_obj_name(name_buffer, sizeof(name_buffer), "xnet://%s"); - (Arg::Gds(isc_network_error) << Arg::Str(name_buffer)).raise(); - } + return false; system_error::raise(ERR_STR("OpenMutex")); } @@ -1123,24 +1120,28 @@ rem_port* XnetClientEndPoint::connect_client(PACKET* packet, const RefPtrgetIpcName(), sizeof(xnet_endpoint)); - - try + if (*xnet_endpoint == 0 || !connect_init()) { - connect_init(); - } - catch (const Exception&) - { - // The client may not have permissions to create global objects, - // but still be able to connect to a local server that has such permissions. - // This is why we try to connect using Global\ namespace unconditionally - fb_utils::snprintf(xnet_endpoint, sizeof(xnet_endpoint), "Global\\%s", conf->getIpcName()); + // First, try to connect using default kernel namespace. + // This should work on Win9X, NT4 and on later OS when server is running + // under restricted account in the same session as the client + fb_utils::copy_terminate(xnet_endpoint, conf->getIpcName(), sizeof(xnet_endpoint)); - if (!connect_init()) { - return NULL; + if (!connect_init()) + { + // The client may not have permissions to create global objects, + // but still be able to connect to a local server that has such permissions. + // This is why we try to connect using Global\ namespace unconditionally + fb_utils::snprintf(xnet_endpoint, sizeof(xnet_endpoint), "Global\\%s", conf->getIpcName()); + + if (!connect_init()) + { + TEXT name_buffer[BUFFER_TINY]; + make_obj_name(name_buffer, sizeof(name_buffer), "xnet://%s"); + + *xnet_endpoint = 0; + (Arg::Gds(isc_network_error) << Arg::Str(name_buffer)).raise(); + } } } From e6d269492ac22c97d9443960c931ead4f1e7d30a Mon Sep 17 00:00:00 2001 From: Alex Peshkoff Date: Fri, 13 Sep 2019 20:31:05 +0300 Subject: [PATCH 030/274] Fixed CORE-6143: Error 'Multiple maps found for ...' is raised in not appropriate case --- src/jrd/Mapping.cpp | 26 +++++++++++++++++--------- src/jrd/Mapping.h | 19 ++++++++++++------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/jrd/Mapping.cpp b/src/jrd/Mapping.cpp index 3950ce71e8..781052398c 100644 --- a/src/jrd/Mapping.cpp +++ b/src/jrd/Mapping.cpp @@ -414,7 +414,7 @@ bool Mapping::Cache::populate(IAttachment *att) return false; } -void Mapping::Cache::map(bool flagWild, AuthReader::Info& info, AuthWriter& newBlock) +void Mapping::Cache::map(bool flagWild, ExtInfo& info, AuthWriter& newBlock) { if (info.type == TYPE_SEEN) return; @@ -430,7 +430,7 @@ void Mapping::Cache::map(bool flagWild, AuthReader::Info& info, AuthWriter& newB varUsing(info, from, newBlock); } -void Mapping::Cache::search(AuthReader::Info& info, const Map& from, AuthWriter& newBlock, +void Mapping::Cache::search(ExtInfo& info, const Map& from, AuthWriter& newBlock, const NoCaseString& originalUserName) { MAP_DEBUG(fprintf(stderr, "Key = %s\n", from.makeHashKey().c_str())); @@ -443,21 +443,29 @@ void Mapping::Cache::search(AuthReader::Info& info, const Map& from, AuthWriter& unsigned flagRolUsr = to->toRole ? FLAG_ROLE : FLAG_USER; if (info.found & flagRolUsr) continue; + + const NoCaseString& newName(to->to == "*" ? originalUserName : to->to); + NoCaseString& infoName(to->toRole ? info.currentRole : info.currentUser); if (info.current & flagRolUsr) + { + if (infoName == newName) + continue; (Arg::Gds(isc_map_multi) << originalUserName).raise(); + } info.current |= flagRolUsr; + infoName = newName; AuthReader::Info newInfo; newInfo.type = to->toRole ? NM_ROLE : NM_USER; - newInfo.name = to->to == "*" ? originalUserName : to->to; + newInfo.name = newName; newInfo.secDb = this->name; newInfo.origPlug = info.origPlug.hasData() ? info.origPlug : info.plugin; newBlock.add(newInfo); } } -void Mapping::Cache::varPlugin(AuthReader::Info& info, Map from, AuthWriter& newBlock) +void Mapping::Cache::varPlugin(ExtInfo& info, Map from, AuthWriter& newBlock) { varDb(info, from, newBlock); if (from.plugin != "*") @@ -467,7 +475,7 @@ void Mapping::Cache::varPlugin(AuthReader::Info& info, Map from, AuthWriter& new } } -void Mapping::Cache::varDb(AuthReader::Info& info, Map from, AuthWriter& newBlock) +void Mapping::Cache::varDb(ExtInfo& info, Map from, AuthWriter& newBlock) { varFrom(info, from, newBlock); if (from.db != "*") @@ -477,7 +485,7 @@ void Mapping::Cache::varDb(AuthReader::Info& info, Map from, AuthWriter& newBloc } } -void Mapping::Cache::varFrom(AuthReader::Info& info, Map from, AuthWriter& newBlock) +void Mapping::Cache::varFrom(ExtInfo& info, Map from, AuthWriter& newBlock) { NoCaseString originalUserName = from.from; search(info, from, newBlock, originalUserName); @@ -485,7 +493,7 @@ void Mapping::Cache::varFrom(AuthReader::Info& info, Map from, AuthWriter& newBl search(info, from, newBlock, originalUserName); } -void Mapping::Cache::varUsing(AuthReader::Info& info, Map from, AuthWriter& newBlock) +void Mapping::Cache::varUsing(ExtInfo& info, Map from, AuthWriter& newBlock) { if (from.usng == 'P') { @@ -512,7 +520,7 @@ void Mapping::Cache::varUsing(AuthReader::Info& info, Map from, AuthWriter& newB fb_assert(false); } -bool Mapping::Cache::map4(bool flagWild, unsigned flagSet, AuthReader& rdr, AuthReader::Info& info, AuthWriter& newBlock) +bool Mapping::Cache::map4(bool flagWild, unsigned flagSet, AuthReader& rdr, ExtInfo& info, AuthWriter& newBlock) { if (!flagSet) { @@ -1439,7 +1447,7 @@ ULONG Mapping::mapUser(string& name, string& trustedRole) // Map it only when needed if (authBlock && authBlock->hasData() && (dbCache || secCache)) { - AuthReader::Info info; + ExtInfo info; // Caches are ready somehow - proceed with analysis AuthReader auth(*authBlock); diff --git a/src/jrd/Mapping.h b/src/jrd/Mapping.h index 7c3f2aeca1..64dccddd30 100644 --- a/src/jrd/Mapping.h +++ b/src/jrd/Mapping.h @@ -98,6 +98,11 @@ private: const Firebird::string* sqlRole; public: + struct ExtInfo : public Firebird::AuthReader::Info + { + Firebird::NoCaseString currentRole, currentUser; + }; + class DbHandle : public Firebird::RefPtr { public: @@ -136,15 +141,15 @@ public: ~Cache(); bool populate(Firebird::IAttachment *att); - void map(bool flagWild, Firebird::AuthReader::Info& info, AuthWriter& newBlock); - void search(Firebird::AuthReader::Info& info, const Map& from, AuthWriter& newBlock, + void map(bool flagWild, ExtInfo& info, AuthWriter& newBlock); + void search(ExtInfo& info, const Map& from, AuthWriter& newBlock, const Firebird::NoCaseString& originalUserName); - void varPlugin(Firebird::AuthReader::Info& info, Map from, AuthWriter& newBlock); - void varDb(Firebird::AuthReader::Info& info, Map from, AuthWriter& newBlock); - void varFrom(Firebird::AuthReader::Info& info, Map from, AuthWriter& newBlock); - void varUsing(Firebird::AuthReader::Info& info, Map from, AuthWriter& newBlock); + void varPlugin(ExtInfo& info, Map from, AuthWriter& newBlock); + void varDb(ExtInfo& info, Map from, AuthWriter& newBlock); + void varFrom(ExtInfo& info, Map from, AuthWriter& newBlock); + void varUsing(ExtInfo& info, Map from, AuthWriter& newBlock); bool map4(bool flagWild, unsigned flagSet, Firebird::AuthReader& rdr, - Firebird::AuthReader::Info& info, AuthWriter& newBlock); + ExtInfo& info, AuthWriter& newBlock); static void eraseEntry(Map* m); public: From 831e9953349d80d1ac0e052138cc429579e14e7d Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 14 Sep 2019 00:03:49 +0000 Subject: [PATCH 031/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index de608c2bde..e62bedc2c8 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1598 + FORMAL BUILD NUMBER:1600 */ -#define PRODUCT_VER_STRING "4.0.0.1598" -#define FILE_VER_STRING "WI-T4.0.0.1598" -#define LICENSE_VER_STRING "WI-T4.0.0.1598" -#define FILE_VER_NUMBER 4, 0, 0, 1598 +#define PRODUCT_VER_STRING "4.0.0.1600" +#define FILE_VER_STRING "WI-T4.0.0.1600" +#define LICENSE_VER_STRING "WI-T4.0.0.1600" +#define FILE_VER_NUMBER 4, 0, 0, 1600 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1598" +#define FB_BUILD_NO "1600" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index ea97da7e53..cde8977493 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1598 +BuildNum=1600 NowAt=`pwd` cd `dirname $0` From 4a322b3b81922486bb0015db0aaadc6c2899a271 Mon Sep 17 00:00:00 2001 From: hvlad Date: Sat, 14 Sep 2019 14:25:17 +0300 Subject: [PATCH 032/274] Fixed bug CORE-6138 : Inconsistent behavior regarding visibility of master record on detail inserts Dmitry, please review --- src/jrd/idx.cpp | 134 ++++++++++++++++++++++++++++++------------------ 1 file changed, 85 insertions(+), 49 deletions(-) diff --git a/src/jrd/idx.cpp b/src/jrd/idx.cpp index 21fa458585..aaf75f6062 100644 --- a/src/jrd/idx.cpp +++ b/src/jrd/idx.cpp @@ -82,6 +82,7 @@ struct index_fast_load static idx_e check_duplicates(thread_db*, Record*, index_desc*, index_insertion*, jrd_rel*); static idx_e check_foreign_key(thread_db*, Record*, jrd_rel*, jrd_tra*, index_desc*, IndexErrorContext&); static idx_e check_partner_index(thread_db*, jrd_rel*, Record*, jrd_tra*, index_desc*, jrd_rel*, USHORT); +static bool cmpRecordKeys(thread_db*, Record*, jrd_rel*, index_desc*, Record*, jrd_rel*, index_desc*); static bool duplicate_key(const UCHAR*, const UCHAR*, void*); static PageNumber get_root_page(thread_db*, jrd_rel*); static int index_block_flush(void*); @@ -1038,6 +1039,67 @@ void IDX_store(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) } } +static bool cmpRecordKeys(thread_db* tdbb, + Record* rec1, jrd_rel* rel1, index_desc* idx1, + Record* rec2, jrd_rel* rel2, index_desc* idx2) +{ + Firebird::HalfStaticArray tmp; + DSC desc1, desc2; + + if (idx2->idx_flags & idx_expressn) + { + // Remove assertion below if\when expression index will participate in FK, + // currently it is impossible. + fb_assert((idx1->idx_flags & idx_expressn) != 0); + + bool flag_idx; + const dsc* desc_idx = BTR_eval_expression(tdbb, idx2, rec2, flag_idx); + + // hvlad: BTR_eval_expression call EVL_expr which returns impure->vlu_desc. + // Since idx2 and idx1 are the same indexes second call to + // BTR_eval_expression will overwrite value from first call. So we must + // save first result into another dsc + + desc1 = *desc_idx; + const USHORT idx_dsc_length = idx2->idx_expression_desc.dsc_length; + desc1.dsc_address = tmp.getBuffer(idx_dsc_length + FB_DOUBLE_ALIGN); + desc1.dsc_address = FB_ALIGN(desc1.dsc_address, FB_DOUBLE_ALIGN); + fb_assert(desc_idx->dsc_length <= idx_dsc_length); + memmove(desc1.dsc_address, desc_idx->dsc_address, desc_idx->dsc_length); + + bool flag_rec = false; + const dsc* desc_rec = BTR_eval_expression(tdbb, idx1, rec1, flag_rec); + + if (flag_rec && flag_idx && (MOV_compare(tdbb, desc_rec, &desc1) == 0)) + return true; + } + else + { + bool all_nulls = true; + USHORT i; + for (i = 0; i < idx1->idx_count; i++) + { + USHORT field_id = idx1->idx_rpt[i].idx_field; + // In order to "map a null to a default" value (in EVL_field()), + // the relation block is referenced. + // Reference: Bug 10116, 10424 + const bool flag_rec = EVL_field(rel1, rec1, field_id, &desc1); + + field_id = idx2->idx_rpt[i].idx_field; + const bool flag_idx = EVL_field(rel2, rec2, field_id, &desc2); + + if (flag_rec != flag_idx || (flag_rec && (MOV_compare(tdbb, &desc1, &desc2) != 0))) + break; + + all_nulls = all_nulls && !flag_rec && !flag_idx; + } + + if (i >= idx1->idx_count && !all_nulls) + return true; + } + + return false; +} static idx_e check_duplicates(thread_db* tdbb, Record* record, @@ -1060,13 +1122,13 @@ static idx_e check_duplicates(thread_db* tdbb, SET_TDBB(tdbb); idx_e result = idx_e_ok; + jrd_tra* const transaction = insertion->iib_transaction; index_desc* insertion_idx = insertion->iib_descriptor; record_param rpb; rpb.rpb_relation = insertion->iib_relation; rpb.rpb_record = NULL; jrd_rel* const relation_1 = insertion->iib_relation; - Firebird::HalfStaticArray tmp; RecordBitmap::Accessor accessor(insertion->iib_duplicates); fb_assert(!(tdbb->tdbb_status_vector->getState() & IStatus::STATE_ERRORS)); @@ -1079,7 +1141,7 @@ static idx_e check_duplicates(thread_db* tdbb, rpb.rpb_number.setValue(accessor.current()); if (rpb.rpb_number != insertion->iib_number && - VIO_get_current(tdbb, &rpb, insertion->iib_transaction, tdbb->getDefaultPool(), + VIO_get_current(tdbb, &rpb, transaction, tdbb->getDefaultPool(), is_fk, rec_tx_active) ) { // hvlad: if record's transaction is still active, we should consider @@ -1095,60 +1157,34 @@ static idx_e check_duplicates(thread_db* tdbb, // record retrieved -- for unique indexes the insertion index and the // record index are the same, but for foreign keys they are different - if (record_idx->idx_flags & idx_expressn) + if (cmpRecordKeys(tdbb, rpb.rpb_record, relation_1, insertion_idx, + record, relation_2, record_idx)) { - bool flag_idx; - const dsc* desc_idx = BTR_eval_expression(tdbb, record_idx, record, flag_idx); + // When check foreign keys in snapshot or read consistency transaction, + // ensure that master record is visible in transaction context and still + // satisfy foreign key constraint. - // hvlad: BTR_eval_expression call EVL_expr which returns impure->vlu_desc. - // Since record_idx and insertion_idx are the same indexes second call to - // BTR_eval_expression will overwrite value from first call. So we must - // save first result into another dsc - - desc1 = *desc_idx; - const USHORT idx_dsc_length = record_idx->idx_expression_desc.dsc_length; - desc1.dsc_address = tmp.getBuffer(idx_dsc_length + FB_DOUBLE_ALIGN); - desc1.dsc_address = FB_ALIGN(desc1.dsc_address, FB_DOUBLE_ALIGN); - fb_assert(desc_idx->dsc_length <= idx_dsc_length); - memmove(desc1.dsc_address, desc_idx->dsc_address, desc_idx->dsc_length); - - bool flag_rec = false; - const dsc* desc_rec = BTR_eval_expression(tdbb, insertion_idx, rpb.rpb_record, flag_rec); - - if (flag_rec && flag_idx && (MOV_compare(tdbb, desc_rec, &desc1) == 0)) + if (is_fk && + (!(transaction->tra_flags & TRA_read_committed) || + (transaction->tra_flags & TRA_read_consistency))) { + const int state = TRA_snapshot_state(tdbb, transaction, rpb.rpb_transaction_nr); + + if (state != tra_committed && state != tra_us) + { + if (!VIO_get(tdbb, &rpb, transaction, tdbb->getDefaultPool())) + continue; + + if (!cmpRecordKeys(tdbb, rpb.rpb_record, relation_1, insertion_idx, + record, relation_2, record_idx)) + continue; + } + } + result = idx_e_duplicate; break; } } - else - { - bool all_nulls = true; - USHORT i; - for (i = 0; i < insertion_idx->idx_count; i++) - { - USHORT field_id = insertion_idx->idx_rpt[i].idx_field; - // In order to "map a null to a default" value (in EVL_field()), - // the relation block is referenced. - // Reference: Bug 10116, 10424 - const bool flag_rec = EVL_field(relation_1, rpb.rpb_record, field_id, &desc1); - - field_id = record_idx->idx_rpt[i].idx_field; - const bool flag_idx = EVL_field(relation_2, record, field_id, &desc2); - - if (flag_rec != flag_idx || (flag_rec && (MOV_compare(tdbb, &desc1, &desc2) != 0) )) - break; - - all_nulls = all_nulls && !flag_rec && !flag_idx; - } - - if (i >= insertion_idx->idx_count && !all_nulls) - { - result = idx_e_duplicate; - break; - } - } - } } while (accessor.getNext()); delete rpb.rpb_record; From 0037fdcb84832f51e0ece6b2cd343d805bd2aa6e Mon Sep 17 00:00:00 2001 From: hvlad Date: Sat, 14 Sep 2019 15:50:00 +0300 Subject: [PATCH 033/274] No need to evaluate same key again. It was necessary before introducing temporary_key::key_nulls in v3. --- src/jrd/idx.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/jrd/idx.cpp b/src/jrd/idx.cpp index aaf75f6062..dffbfa995a 100644 --- a/src/jrd/idx.cpp +++ b/src/jrd/idx.cpp @@ -1544,11 +1544,7 @@ static idx_e insert_key(thread_db* tdbb, { // Find out if there is a null segment. If there is one, // don't bother to check the primary key. - CCH_FETCH(tdbb, window_ptr, LCK_read, pag_root); - temporary_key key; - result = BTR_key(tdbb, relation, record, idx, &key, false); - CCH_RELEASE(tdbb, window_ptr); - if (result == idx_e_ok && key.key_nulls == 0) + if (result == idx_e_ok && insertion->iib_key->key_nulls == 0) { result = check_foreign_key(tdbb, record, insertion->iib_relation, transaction, idx, context); From b7c31c7ac1c3554b3c600be2c6b2c86a03fc6e79 Mon Sep 17 00:00:00 2001 From: hvlad Date: Sat, 14 Sep 2019 15:57:40 +0300 Subject: [PATCH 034/274] Comments, misc --- src/jrd/idx.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/jrd/idx.cpp b/src/jrd/idx.cpp index dffbfa995a..682506919e 100644 --- a/src/jrd/idx.cpp +++ b/src/jrd/idx.cpp @@ -1043,7 +1043,20 @@ static bool cmpRecordKeys(thread_db* tdbb, Record* rec1, jrd_rel* rel1, index_desc* idx1, Record* rec2, jrd_rel* rel2, index_desc* idx2) { - Firebird::HalfStaticArray tmp; +/************************************** + * + * c m p R e c o r d K e y s + * + ************************************** + * + * Functional description + * Compare indexed fields in two records. Records could belong to different + * relations but set of indexed fields to compare should be equal. + * + **************************************/ + SET_TDBB(tdbb); + + HalfStaticArray tmp; DSC desc1, desc2; if (idx2->idx_flags & idx_expressn) @@ -1075,6 +1088,8 @@ static bool cmpRecordKeys(thread_db* tdbb, } else { + fb_assert(idx1->idx_count == idx2->idx_count); + bool all_nulls = true; USHORT i; for (i = 0; i < idx1->idx_count; i++) From b26574364241e79f6b3151bccec9b60e4b4053f5 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sun, 15 Sep 2019 00:03:27 +0000 Subject: [PATCH 035/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index e62bedc2c8..7cb6de1964 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1600 + FORMAL BUILD NUMBER:1603 */ -#define PRODUCT_VER_STRING "4.0.0.1600" -#define FILE_VER_STRING "WI-T4.0.0.1600" -#define LICENSE_VER_STRING "WI-T4.0.0.1600" -#define FILE_VER_NUMBER 4, 0, 0, 1600 +#define PRODUCT_VER_STRING "4.0.0.1603" +#define FILE_VER_STRING "WI-T4.0.0.1603" +#define LICENSE_VER_STRING "WI-T4.0.0.1603" +#define FILE_VER_NUMBER 4, 0, 0, 1603 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1600" +#define FB_BUILD_NO "1603" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index cde8977493..3a1dc0c9da 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1600 +BuildNum=1603 NowAt=`pwd` cd `dirname $0` From 861d536fc26542335f0fb84cdea825e891364057 Mon Sep 17 00:00:00 2001 From: Alexander Peshkov Date: Mon, 16 Sep 2019 20:59:54 +0300 Subject: [PATCH 036/274] Int128 - new datatype (#220) * Int128 support - work in progress * Work in progress * Int128 datatype appears to be mostly OK except sort & index * Fixed divide scaling, added sorting & network (xdr) support * Binding control, aggregate nodes, cleanup and documentation * Fixed VS2017 AppVeyor build * Next attempt to fix vs2017 build * Next attempt to fix vs2017 build * Next attempt to fix vs2017 build * Update MSVC build. * Set VS architecture correctly * Fixed a number of issues noticed by Mark --- appveyor.yml | 4 +- builds/win32/make_boot.bat | 20 + builds/win32/msvc10/common.vcxproj | 6 +- builds/win32/msvc10/common.vcxproj.filters | 6 + builds/win32/msvc12/common.vcxproj | 6 +- builds/win32/msvc12/common.vcxproj.filters | 6 + builds/win32/msvc14/common.vcxproj | 6 +- builds/win32/msvc14/common.vcxproj.filters | 6 + builds/win32/msvc15/common.vcxproj | 6 +- builds/win32/msvc15/common.vcxproj.filters | 6 + doc/sql.extensions/README.data_types | 31 +- extern/ttmath/ttmath.h | 2843 ++++++++++++ extern/ttmath/ttmathint.h | 1917 ++++++++ extern/ttmath/ttmathmisc.h | 250 ++ extern/ttmath/ttmathobjects.h | 809 ++++ extern/ttmath/ttmaththreads.h | 250 ++ extern/ttmath/ttmathtypes.h | 676 +++ extern/ttmath/ttmathuint.h | 4126 ++++++++++++++++++ extern/ttmath/ttmathuint_noasm.h | 1017 +++++ extern/ttmath/ttmathuint_x86.h | 1602 +++++++ extern/ttmath/ttmathuint_x86_64.h | 1146 +++++ extern/ttmath/ttmathuint_x86_64_msvc.asm | 548 +++ src/burp/backup.epp | 4 +- src/burp/canonical.cpp | 6 +- src/burp/restore.epp | 2 +- src/common/DecFloat.cpp | 249 +- src/common/DecFloat.h | 132 +- src/common/Int128.cpp | 492 +++ src/common/Int128.h | 147 + src/common/classes/InternalMessageBuffer.cpp | 6 +- src/common/classes/fb_string.h | 10 +- src/common/cvt.cpp | 440 +- src/common/cvt.h | 10 +- src/common/dsc.cpp | 112 +- src/common/dsc.h | 26 +- src/common/keywords.cpp | 1 + src/common/sdl.cpp | 6 +- src/common/utils.cpp | 4 +- src/common/xdr.cpp | 21 +- src/common/xdr_proto.h | 5 +- src/dsql/AggNodes.cpp | 39 +- src/dsql/DdlNodes.epp | 10 +- src/dsql/ExprNodes.cpp | 173 +- src/dsql/Nodes.h | 2 +- src/dsql/StmtNodes.cpp | 9 +- src/dsql/StmtNodes.h | 14 +- src/dsql/ddl_proto.h | 2 +- src/dsql/dsql.cpp | 8 +- src/dsql/dsql.h | 4 +- src/dsql/gen.cpp | 21 +- src/dsql/movd.cpp | 4 +- src/dsql/movd_proto.h | 2 +- src/dsql/parse.y | 56 +- src/include/firebird/FirebirdInterface.idl | 15 +- src/include/firebird/IdlFbInterfaces.h | 141 +- src/include/firebird/impl/blr.h | 2 +- src/include/firebird/impl/consts_pub.h | 1 + src/include/firebird/impl/dsc_pub.h | 2 +- src/include/firebird/impl/sqlda_pub.h | 2 +- src/include/firebird/impl/types_pub.h | 5 + src/isql/isql.epp | 88 +- src/isql/isql.h | 5 +- src/jrd/Attachment.cpp | 2 +- src/jrd/Attachment.h | 8 +- src/jrd/ExtEngineManager.cpp | 3 - src/jrd/PreparedStatement.cpp | 6 +- src/jrd/Routine.cpp | 2 +- src/jrd/align.h | 9 +- src/jrd/cvt.cpp | 21 +- src/jrd/cvt2.cpp | 18 +- src/jrd/dfw.epp | 2 +- src/jrd/evl.cpp | 4 +- src/jrd/exe.h | 5 +- src/jrd/fun.epp | 21 +- src/jrd/jrd.cpp | 81 +- src/jrd/mov.cpp | 40 +- src/jrd/mov_proto.h | 3 +- src/jrd/opt.cpp | 2 +- src/jrd/par.cpp | 6 +- src/jrd/sort.cpp | 19 + src/jrd/sort.h | 1 + src/jrd/val.h | 14 +- src/misc/pascal/Pascal.interface.pas | 2 +- src/remote/client/BlrFromMessage.cpp | 6 +- src/remote/parser.cpp | 8 +- src/utilities/ntrace/TracePluginImpl.cpp | 7 +- src/yvalve/YObjects.h | 1 + src/yvalve/gds.cpp | 9 +- src/yvalve/utl.cpp | 43 + 89 files changed, 17082 insertions(+), 826 deletions(-) create mode 100644 extern/ttmath/ttmath.h create mode 100644 extern/ttmath/ttmathint.h create mode 100644 extern/ttmath/ttmathmisc.h create mode 100644 extern/ttmath/ttmathobjects.h create mode 100644 extern/ttmath/ttmaththreads.h create mode 100644 extern/ttmath/ttmathtypes.h create mode 100644 extern/ttmath/ttmathuint.h create mode 100644 extern/ttmath/ttmathuint_noasm.h create mode 100644 extern/ttmath/ttmathuint_x86.h create mode 100644 extern/ttmath/ttmathuint_x86_64.h create mode 100644 extern/ttmath/ttmathuint_x86_64_msvc.asm create mode 100644 src/common/Int128.cpp create mode 100644 src/common/Int128.h diff --git a/appveyor.yml b/appveyor.yml index 99f821570d..fa9564dd4c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,9 +27,11 @@ install: - cmd: for /r %%i in (*.bat) do unix2dos "%%i" - cmd: if "%PLATFORM%" == "x64" set FB_PROCESSOR_ARCHITECTURE=AMD64 - cmd: if "%PLATFORM%" == "x64" set FB_OUTPUT_SUFFIX=x64 + - cmd: if "%PLATFORM%" == "x64" set FB_VS_ARCH=amd64 - cmd: if "%PLATFORM%" == "x86" set FB_PROCESSOR_ARCHITECTURE=x86 - cmd: if "%PLATFORM%" == "x86" set FB_OUTPUT_SUFFIX=win32 - - cmd: if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" call "%ProgramFiles(x86)%\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" + - cmd: if "%PLATFORM%" == "x86" set FB_VS_ARCH=x86 + - cmd: if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" call "%ProgramFiles(x86)%\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=%FB_VS_ARCH% - cmd: cd builds\win32 - cmd: run_all.bat JUSTBUILD - cmd: set ARTIFACTS_PATH=output_%FB_OUTPUT_SUFFIX% diff --git a/builds/win32/make_boot.bat b/builds/win32/make_boot.bat index b4d62b90dd..882c0829e7 100644 --- a/builds/win32/make_boot.bat +++ b/builds/win32/make_boot.bat @@ -41,6 +41,9 @@ if "%ERRLEV%"=="1" goto :END call :decNumber if "%ERRLEV%"=="1" goto :END +if "%FB_TARGET_PLATFORM%"=="x64" call :ttmath +if "%ERRLEV%"=="1" goto :END + call :re2 if "%ERRLEV%"=="1" goto :END @@ -162,6 +165,23 @@ if errorlevel 1 call :boot2 decNumber_%FB_OBJ_DIR% @call set_build_target.bat %* goto :EOF +::=================== +:: BUILD ttmath +:ttmath +@echo. +@call set_build_target.bat %* RELEASE +@echo Building ttmath (%FB_OBJ_DIR%)... +@mkdir %FB_TEMP_DIR%\..\%FB_OBJ_DIR%\common 2>nul +@ml64.exe /c /Fo %FB_TEMP_DIR%\..\%FB_OBJ_DIR%\common\ttmathuint_x86_64_msvc.obj %FB_ROOT_PATH%\extern\ttmath\ttmathuint_x86_64_msvc.asm +if errorlevel 1 call :boot2 ttmath_%FB_OBJ_DIR% +@call set_build_target.bat %* DEBUG +@echo Building decNumber (%FB_OBJ_DIR%)... +@mkdir %FB_TEMP_DIR%\..\%FB_OBJ_DIR%\common 2>nul +@ml64.exe /c /Zi /Fo %FB_TEMP_DIR%\..\%FB_OBJ_DIR%\common\ttmathuint_x86_64_msvc.obj %FB_ROOT_PATH%\extern\ttmath\ttmathuint_x86_64_msvc.asm +if errorlevel 1 call :boot2 ttmath_%FB_OBJ_DIR% +@call set_build_target.bat %* +goto :EOF + ::=================== :: BUILD re2 :re2 diff --git a/builds/win32/msvc10/common.vcxproj b/builds/win32/msvc10/common.vcxproj index 99df722d88..f8f9271413 100644 --- a/builds/win32/msvc10/common.vcxproj +++ b/builds/win32/msvc10/common.vcxproj @@ -72,6 +72,7 @@ + @@ -180,6 +181,7 @@ + @@ -335,7 +337,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) @@ -353,7 +355,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) diff --git a/builds/win32/msvc10/common.vcxproj.filters b/builds/win32/msvc10/common.vcxproj.filters index 978f26a315..f6d9bf98c0 100644 --- a/builds/win32/msvc10/common.vcxproj.filters +++ b/builds/win32/msvc10/common.vcxproj.filters @@ -240,6 +240,9 @@ common + + common + @@ -581,5 +584,8 @@ headers + + headers + \ No newline at end of file diff --git a/builds/win32/msvc12/common.vcxproj b/builds/win32/msvc12/common.vcxproj index 0492407080..8b2e9805a0 100644 --- a/builds/win32/msvc12/common.vcxproj +++ b/builds/win32/msvc12/common.vcxproj @@ -68,6 +68,7 @@ + @@ -176,6 +177,7 @@ + @@ -335,7 +337,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) @@ -353,7 +355,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) diff --git a/builds/win32/msvc12/common.vcxproj.filters b/builds/win32/msvc12/common.vcxproj.filters index 328eabca1d..ffc6d9c8fe 100644 --- a/builds/win32/msvc12/common.vcxproj.filters +++ b/builds/win32/msvc12/common.vcxproj.filters @@ -240,6 +240,9 @@ common + + common + @@ -581,5 +584,8 @@ headers + + headers + \ No newline at end of file diff --git a/builds/win32/msvc14/common.vcxproj b/builds/win32/msvc14/common.vcxproj index 77823eb058..615a314773 100644 --- a/builds/win32/msvc14/common.vcxproj +++ b/builds/win32/msvc14/common.vcxproj @@ -68,6 +68,7 @@ + @@ -177,6 +178,7 @@ + @@ -337,7 +339,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) @@ -355,7 +357,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) diff --git a/builds/win32/msvc14/common.vcxproj.filters b/builds/win32/msvc14/common.vcxproj.filters index a0b2d29362..4da18bbba5 100644 --- a/builds/win32/msvc14/common.vcxproj.filters +++ b/builds/win32/msvc14/common.vcxproj.filters @@ -243,6 +243,9 @@ common + + common + @@ -587,5 +590,8 @@ headers + + headers + diff --git a/builds/win32/msvc15/common.vcxproj b/builds/win32/msvc15/common.vcxproj index c2ccd9f972..e8d6f56042 100644 --- a/builds/win32/msvc15/common.vcxproj +++ b/builds/win32/msvc15/common.vcxproj @@ -68,6 +68,7 @@ + @@ -177,6 +178,7 @@ + @@ -338,7 +340,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) @@ -356,7 +358,7 @@ 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) + ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) diff --git a/builds/win32/msvc15/common.vcxproj.filters b/builds/win32/msvc15/common.vcxproj.filters index 0944d0518d..776d4a9db7 100644 --- a/builds/win32/msvc15/common.vcxproj.filters +++ b/builds/win32/msvc15/common.vcxproj.filters @@ -243,6 +243,9 @@ common + + common + @@ -587,5 +590,8 @@ headers + + headers + \ No newline at end of file diff --git a/doc/sql.extensions/README.data_types b/doc/sql.extensions/README.data_types index 4df707dc4c..ac6dbc10b0 100644 --- a/doc/sql.extensions/README.data_types +++ b/doc/sql.extensions/README.data_types @@ -200,7 +200,7 @@ Enhancement in precision of calculations with NUMERIC/DECIMAL (FB 4.0) -------------- Function: - Maximum precision of NUMERIC and DECIMAL data types is increased to 34 digits. + Maximum precision of NUMERIC and DECIMAL data types is increased to 38 digits. Author: Alex Peshkoff @@ -208,20 +208,35 @@ Enhancement in precision of calculations with NUMERIC/DECIMAL (FB 4.0) Syntax rules: NUMERIC ( P {, N} ) DECIMAL ( P {, N} ) - where P is precision (P <= 34, was limited prior with 18 digits) and N is optional number + where P is precision (P <= 38, was limited prior with 18 digits) and N is optional number of digits after decimal separator (as before). Storage: - 128-bit, format according to IEEE 754. + 128-bit signed integer. Example(s): 1. DECLARE VARIABLE VAR1 DECIMAL(25); - 2. CREATE TABLE TABLE1 (FIELD1 NUMERIC(34, 17)); + 2. CREATE TABLE TABLE1 (FIELD1 NUMERIC(38, 19)); Note(s): Numerics with precision less than 19 digits use SMALLINT, INTEGER, BIGINT or DOUBLE PRECISION as base datatype depending upon number of digits and dialect. When precision is between 19 and - 34 digits DECFLOAT(34) is used for it. Actual precision is always increased to 34 digits. For - complex calculations such digits are casted (internally, in trivial way) to DECFLOAT(34) and - the result of various math (log, exp, etc.) and aggregate functions using high precision - numeric argument is DECFLOAT(34). + 38 digits 128-bit integer is used for it. Actual precision is always increased to 38 digits. + For complex calculations such digits are casted (internally) to DECFLOAT(34) and the result of + various math (log, exp, etc.) and aggregate functions using high precision numeric argument is + DECFLOAT(34). + + SET INT128 BIND - controls how are INT128 values represented in outer + world (i.e. in messages or in XSQLDA). Valid binding types are: NATIVE (use 128-bit + binary representation), CHAR/CHARACTER (use ASCII string), DOUBLE PRECISION (use + 8-byte FP representation - same as used for DOUBLE PRECISION fields) or BIGINT + with possible comma-separated SCALE clause (i.e. 'BIGINT, 3'). Various bindings + are useful if one plans to use 128-bit integers with some old client not supporting + native format. One can choose between strings (ideal precision, but poor support + for further processing), floating point values (ideal support for further processing + but poor precision) or scaled integers (good support for further processing and + required precision but range of values is very limited). When using in a tool like + generic purporse GUI client choice of CHAR binding is OK in most cases. By default + NATIVE binding is used. + The initial configuration may be specified with DPB isc_dpb_int128_bind followed + by a string with its value (case does not matter). diff --git a/extern/ttmath/ttmath.h b/extern/ttmath/ttmath.h new file mode 100644 index 0000000000..9b9e8ee6e5 --- /dev/null +++ b/extern/ttmath/ttmath.h @@ -0,0 +1,2843 @@ +/* + * This file is a part of TTMath Bignum Library + * and is distributed under the (new) BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2006-2012, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name Tomasz Sowa nor the names of contributors to this + * project may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + + + +#ifndef headerfilettmathmathtt +#define headerfilettmathmathtt + +/*! + \file ttmath.h + \brief Mathematics functions. +*/ + +#ifdef _MSC_VER +//warning C4127: conditional expression is constant +#pragma warning( disable: 4127 ) +//warning C4702: unreachable code +#pragma warning( disable: 4702 ) +//warning C4800: forcing value to bool 'true' or 'false' (performance warning) +#pragma warning( disable: 4800 ) +#endif + +#define TTMATH_DONT_USE_WCHAR + +#include "ttmathint.h" +#include "ttmathobjects.h" + + +namespace ttmath +{ + /* + * + * functions defined here are used only with Big<> types + * + * + */ + + + /* + * + * functions for rounding + * + * + */ + + + /*! + this function skips the fraction from x + e.g 2.2 = 2 + 2.7 = 2 + -2.2 = 2 + -2.7 = 2 + */ + template + ValueType SkipFraction(const ValueType & x) + { + ValueType result( x ); + result.SkipFraction(); + + return result; + } + + + /*! + this function rounds to the nearest integer value + e.g 2.2 = 2 + 2.7 = 3 + -2.2 = -2 + -2.7 = -3 + */ + template + ValueType Round(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType result( x ); + uint c = result.Round(); + + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + + /*! + this function returns a value representing the smallest integer + that is greater than or equal to x + + Ceil(-3.7) = -3 + Ceil(-3.1) = -3 + Ceil(-3.0) = -3 + Ceil(4.0) = 4 + Ceil(4.2) = 5 + Ceil(4.8) = 5 + */ + template + ValueType Ceil(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType result(x); + uint c = 0; + + result.SkipFraction(); + + if( result != x ) + { + // x is with fraction + // if x is negative we don't have to do anything + if( !x.IsSign() ) + { + ValueType one; + one.SetOne(); + + c += result.Add(one); + } + } + + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + /*! + this function returns a value representing the largest integer + that is less than or equal to x + + Floor(-3.6) = -4 + Floor(-3.1) = -4 + Floor(-3) = -3 + Floor(2) = 2 + Floor(2.3) = 2 + Floor(2.8) = 2 + */ + template + ValueType Floor(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType result(x); + uint c = 0; + + result.SkipFraction(); + + if( result != x ) + { + // x is with fraction + // if x is positive we don't have to do anything + if( x.IsSign() ) + { + ValueType one; + one.SetOne(); + + c += result.Sub(one); + } + } + + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + + /* + * + * logarithms and the exponent + * + * + */ + + + /*! + this function calculates the natural logarithm (logarithm with the base 'e') + */ + template + ValueType Ln(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType result; + uint state = result.Ln(x); + + if( err ) + { + switch( state ) + { + case 0: + *err = err_ok; + break; + case 1: + *err = err_overflow; + break; + case 2: + *err = err_improper_argument; + break; + default: + *err = err_internal_error; + break; + } + } + + + return result; + } + + + /*! + this function calculates the logarithm + */ + template + ValueType Log(const ValueType & x, const ValueType & base, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) *err = err_improper_argument; + return x; + } + + if( base.IsNan() ) + { + if( err ) *err = err_improper_argument; + return base; + } + + ValueType result; + uint state = result.Log(x, base); + + if( err ) + { + switch( state ) + { + case 0: + *err = err_ok; + break; + case 1: + *err = err_overflow; + break; + case 2: + case 3: + *err = err_improper_argument; + break; + default: + *err = err_internal_error; + break; + } + } + + return result; + } + + + /*! + this function calculates the expression e^x + */ + template + ValueType Exp(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType result; + uint c = result.Exp(x); + + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + /*! + * + * trigonometric functions + * + */ + + + /* + this namespace consists of auxiliary functions + (something like 'private' in a class) + */ + namespace auxiliaryfunctions + { + + /*! + an auxiliary function for calculating the Sine + (you don't have to call this function) + */ + template + uint PrepareSin(ValueType & x, bool & change_sign) + { + ValueType temp; + + change_sign = false; + + if( x.IsSign() ) + { + // we're using the formula 'sin(-x) = -sin(x)' + change_sign = !change_sign; + x.ChangeSign(); + } + + // we're reducing the period 2*PI + // (for big values there'll always be zero) + temp.Set2Pi(); + + if( x.Mod(temp) ) + return 1; + + + // we're setting 'x' as being in the range of <0, 0.5PI> + + temp.SetPi(); + + if( x > temp ) + { + // x is in (pi, 2*pi> + x.Sub( temp ); + change_sign = !change_sign; + } + + temp.Set05Pi(); + + if( x > temp ) + { + // x is in (0.5pi, pi> + x.Sub( temp ); + x = temp - x; + } + + return 0; + } + + + /*! + an auxiliary function for calculating the Sine + (you don't have to call this function) + + it returns Sin(x) where 'x' is from <0, PI/2> + we're calculating the Sin with using Taylor series in zero or PI/2 + (depending on which point of these two points is nearer to the 'x') + + Taylor series: + sin(x) = sin(a) + cos(a)*(x-a)/(1!) + - sin(a)*((x-a)^2)/(2!) - cos(a)*((x-a)^3)/(3!) + + sin(a)*((x-a)^4)/(4!) + ... + + when a=0 it'll be: + sin(x) = (x)/(1!) - (x^3)/(3!) + (x^5)/(5!) - (x^7)/(7!) + (x^9)/(9!) ... + + and when a=PI/2: + sin(x) = 1 - ((x-PI/2)^2)/(2!) + ((x-PI/2)^4)/(4!) - ((x-PI/2)^6)/(6!) ... + */ + template + ValueType Sin0pi05(const ValueType & x) + { + ValueType result; + ValueType numerator, denominator; + ValueType d_numerator, d_denominator; + ValueType one, temp, old_result; + + // temp = pi/4 + temp.Set05Pi(); + temp.exponent.SubOne(); + + one.SetOne(); + + if( x < temp ) + { + // we're using the Taylor series with a=0 + result = x; + numerator = x; + denominator = one; + + // d_numerator = x^2 + d_numerator = x; + d_numerator.Mul(x); + + d_denominator = 2; + } + else + { + // we're using the Taylor series with a=PI/2 + result = one; + numerator = one; + denominator = one; + + // d_numerator = (x-pi/2)^2 + ValueType pi05; + pi05.Set05Pi(); + + temp = x; + temp.Sub( pi05 ); + d_numerator = temp; + d_numerator.Mul( temp ); + + d_denominator = one; + } + + uint c = 0; + bool addition = false; + + old_result = result; + for(uint i=1 ; i<=TTMATH_ARITHMETIC_MAX_LOOP ; ++i) + { + // we're starting from a second part of the formula + c += numerator. Mul( d_numerator ); + c += denominator. Mul( d_denominator ); + c += d_denominator.Add( one ); + c += denominator. Mul( d_denominator ); + c += d_denominator.Add( one ); + temp = numerator; + c += temp.Div(denominator); + + if( c ) + // Sin is from <-1,1> and cannot make an overflow + // but the carry can be from the Taylor series + // (then we only break our calculations) + break; + + if( addition ) + result.Add( temp ); + else + result.Sub( temp ); + + + addition = !addition; + + // we're testing whether the result has changed after adding + // the next part of the Taylor formula, if not we end the loop + // (it means 'x' is zero or 'x' is PI/2 or this part of the formula + // is too small) + if( result == old_result ) + break; + + old_result = result; + } + + return result; + } + + } // namespace auxiliaryfunctions + + + + /*! + this function calculates the Sine + */ + template + ValueType Sin(ValueType x, ErrorCode * err = 0) + { + using namespace auxiliaryfunctions; + + ValueType one, result; + bool change_sign; + + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; + } + + if( err ) + *err = err_ok; + + if( PrepareSin( x, change_sign ) ) + { + // x is too big, we cannnot reduce the 2*PI period + // prior to version 0.8.5 the result was zero + + // result has NaN flag set by default + + if( err ) + *err = err_overflow; // maybe another error code? err_improper_argument? + + return result; // NaN is set by default + } + + result = Sin0pi05( x ); + + one.SetOne(); + + // after calculations there can be small distortions in the result + if( result > one ) + result = one; + else + if( result.IsSign() ) + // we've calculated the sin from <0, pi/2> and the result + // should be positive + result.SetZero(); + + if( change_sign ) + result.ChangeSign(); + + return result; + } + + + /*! + this function calulates the Cosine + we're using the formula cos(x) = sin(x + PI/2) + */ + template + ValueType Cos(ValueType x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType pi05; + pi05.Set05Pi(); + + uint c = x.Add( pi05 ); + + if( c ) + { + if( err ) + *err = err_overflow; + + return ValueType(); // result is undefined (NaN is set by default) + } + + return Sin(x, err); + } + + + /*! + this function calulates the Tangent + we're using the formula tan(x) = sin(x) / cos(x) + + it takes more time than calculating the Tan directly + from for example Taylor series but should be a bit preciser + because Tan receives its values from -infinity to +infinity + and when we calculate it from any series then we can make + a greater mistake than calculating 'sin/cos' + */ + template + ValueType Tan(const ValueType & x, ErrorCode * err = 0) + { + ValueType result = Cos(x, err); + + if( err && *err != err_ok ) + return result; + + if( result.IsZero() ) + { + if( err ) + *err = err_improper_argument; + + result.SetNan(); + + return result; + } + + return Sin(x, err) / result; + } + + + /*! + this function calulates the Tangent + look at the description of Tan(...) + + (the abbreviation of Tangent can be 'tg' as well) + */ + template + ValueType Tg(const ValueType & x, ErrorCode * err = 0) + { + return Tan(x, err); + } + + + /*! + this function calulates the Cotangent + we're using the formula tan(x) = cos(x) / sin(x) + + (why do we make it in this way? + look at information in Tan() function) + */ + template + ValueType Cot(const ValueType & x, ErrorCode * err = 0) + { + ValueType result = Sin(x, err); + + if( err && *err != err_ok ) + return result; + + if( result.IsZero() ) + { + if( err ) + *err = err_improper_argument; + + result.SetNan(); + + return result; + } + + return Cos(x, err) / result; + } + + + /*! + this function calulates the Cotangent + look at the description of Cot(...) + + (the abbreviation of Cotangent can be 'ctg' as well) + */ + template + ValueType Ctg(const ValueType & x, ErrorCode * err = 0) + { + return Cot(x, err); + } + + + /* + * + * inverse trigonometric functions + * + * + */ + + namespace auxiliaryfunctions + { + + /*! + an auxiliary function for calculating the Arc Sine + + we're calculating asin from the following formula: + asin(x) = x + (1*x^3)/(2*3) + (1*3*x^5)/(2*4*5) + (1*3*5*x^7)/(2*4*6*7) + ... + where abs(x) <= 1 + + we're using this formula when x is from <0, 1/2> + */ + template + ValueType ASin_0(const ValueType & x) + { + ValueType nominator, denominator, nominator_add, nominator_x, denominator_add, denominator_x; + ValueType two, result(x), x2(x); + ValueType nominator_temp, denominator_temp, old_result = result; + uint c = 0; + + x2.Mul(x); + two = 2; + + nominator.SetOne(); + denominator = two; + nominator_add = nominator; + denominator_add = denominator; + nominator_x = x; + denominator_x = 3; + + for(uint i=1 ; i<=TTMATH_ARITHMETIC_MAX_LOOP ; ++i) + { + c += nominator_x.Mul(x2); + nominator_temp = nominator_x; + c += nominator_temp.Mul(nominator); + denominator_temp = denominator; + c += denominator_temp.Mul(denominator_x); + c += nominator_temp.Div(denominator_temp); + + // if there is a carry somewhere we only break the calculating + // the result should be ok -- it's from <-pi/2, pi/2> + if( c ) + break; + + result.Add(nominator_temp); + + if( result == old_result ) + // there's no sense to calculate more + break; + + old_result = result; + + + c += nominator_add.Add(two); + c += denominator_add.Add(two); + c += nominator.Mul(nominator_add); + c += denominator.Mul(denominator_add); + c += denominator_x.Add(two); + } + + return result; + } + + + + /*! + an auxiliary function for calculating the Arc Sine + + we're calculating asin from the following formula: + asin(x) = pi/2 - sqrt(2)*sqrt(1-x) * asin_temp + asin_temp = 1 + (1*(1-x))/((2*3)*(2)) + (1*3*(1-x)^2)/((2*4*5)*(4)) + (1*3*5*(1-x)^3)/((2*4*6*7)*(8)) + ... + + where abs(x) <= 1 + + we're using this formula when x is from (1/2, 1> + */ + template + ValueType ASin_1(const ValueType & x) + { + ValueType nominator, denominator, nominator_add, nominator_x, nominator_x_add, denominator_add, denominator_x; + ValueType denominator2; + ValueType one, two, result; + ValueType nominator_temp, denominator_temp, old_result; + uint c = 0; + + two = 2; + + one.SetOne(); + nominator = one; + result = one; + old_result = result; + denominator = two; + nominator_add = nominator; + denominator_add = denominator; + nominator_x = one; + nominator_x.Sub(x); + nominator_x_add = nominator_x; + denominator_x = 3; + denominator2 = two; + + + for(uint i=1 ; i<=TTMATH_ARITHMETIC_MAX_LOOP ; ++i) + { + nominator_temp = nominator_x; + c += nominator_temp.Mul(nominator); + denominator_temp = denominator; + c += denominator_temp.Mul(denominator_x); + c += denominator_temp.Mul(denominator2); + c += nominator_temp.Div(denominator_temp); + + // if there is a carry somewhere we only break the calculating + // the result should be ok -- it's from <-pi/2, pi/2> + if( c ) + break; + + result.Add(nominator_temp); + + if( result == old_result ) + // there's no sense to calculate more + break; + + old_result = result; + + c += nominator_x.Mul(nominator_x_add); + c += nominator_add.Add(two); + c += denominator_add.Add(two); + c += nominator.Mul(nominator_add); + c += denominator.Mul(denominator_add); + c += denominator_x.Add(two); + c += denominator2.Mul(two); + } + + + nominator_x_add.exponent.AddOne(); // *2 + one.exponent.SubOne(); // =0.5 + nominator_x_add.Pow(one); // =sqrt(nominator_x_add) + result.Mul(nominator_x_add); + + one.Set05Pi(); + one.Sub(result); + + return one; + } + + + } // namespace auxiliaryfunctions + + + /*! + this function calculates the Arc Sine + x is from <-1,1> + */ + template + ValueType ASin(ValueType x, ErrorCode * err = 0) + { + using namespace auxiliaryfunctions; + + ValueType result, one; + one.SetOne(); + bool change_sign = false; + + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; + } + + if( x.GreaterWithoutSignThan(one) ) + { + if( err ) + *err = err_improper_argument; + + return result; // NaN is set by default + } + + if( x.IsSign() ) + { + change_sign = true; + x.Abs(); + } + + one.exponent.SubOne(); // =0.5 + + // asin(-x) = -asin(x) + if( x.GreaterWithoutSignThan(one) ) + result = ASin_1(x); + else + result = ASin_0(x); + + if( change_sign ) + result.ChangeSign(); + + if( err ) + *err = err_ok; + + return result; + } + + + /*! + this function calculates the Arc Cosine + + we're using the formula: + acos(x) = pi/2 - asin(x) + */ + template + ValueType ACos(const ValueType & x, ErrorCode * err = 0) + { + ValueType temp; + + temp.Set05Pi(); + temp.Sub(ASin(x, err)); + + return temp; + } + + + + namespace auxiliaryfunctions + { + + /*! + an auxiliary function for calculating the Arc Tangent + + arc tan (x) where x is in <0; 0.5) + (x can be in (-0.5 ; 0.5) too) + + we're using the Taylor series expanded in zero: + atan(x) = x - (x^3)/3 + (x^5)/5 - (x^7)/7 + ... + */ + template + ValueType ATan0(const ValueType & x) + { + ValueType nominator, denominator, nominator_add, denominator_add, temp; + ValueType result, old_result; + bool adding = false; + uint c = 0; + + result = x; + old_result = result; + nominator = x; + nominator_add = x; + nominator_add.Mul(x); + + denominator.SetOne(); + denominator_add = 2; + + for(uint i=1 ; i<=TTMATH_ARITHMETIC_MAX_LOOP ; ++i) + { + c += nominator.Mul(nominator_add); + c += denominator.Add(denominator_add); + + temp = nominator; + c += temp.Div(denominator); + + if( c ) + // the result should be ok + break; + + if( adding ) + result.Add(temp); + else + result.Sub(temp); + + if( result == old_result ) + // there's no sense to calculate more + break; + + old_result = result; + adding = !adding; + } + + return result; + } + + + /*! + an auxiliary function for calculating the Arc Tangent + + where x is in <0 ; 1> + */ + template + ValueType ATan01(const ValueType & x) + { + ValueType half; + half.Set05(); + + /* + it would be better if we chose about sqrt(2)-1=0.41... instead of 0.5 here + + because as you can see below: + when x = sqrt(2)-1 + abs(x) = abs( (x-1)/(1+x) ) + so when we're calculating values around x + then they will be better converged to each other + + for example if we have x=0.4999 then during calculating ATan0(0.4999) + we have to make about 141 iterations but when we have x=0.5 + then during calculating ATan0( (x-1)/(1+x) ) we have to make + only about 89 iterations (both for Big<3,9>) + + in the future this 0.5 can be changed + */ + if( x.SmallerWithoutSignThan(half) ) + return ATan0(x); + + + /* + x>=0.5 and x<=1 + (x can be even smaller than 0.5) + + y = atac(x) + x = tan(y) + + tan(y-b) = (tan(y)-tab(b)) / (1+tan(y)*tan(b)) + y-b = atan( (tan(y)-tab(b)) / (1+tan(y)*tan(b)) ) + y = b + atan( (x-tab(b)) / (1+x*tan(b)) ) + + let b = pi/4 + tan(b) = tan(pi/4) = 1 + y = pi/4 + atan( (x-1)/(1+x) ) + + so + atac(x) = pi/4 + atan( (x-1)/(1+x) ) + when x->1 (x converges to 1) the (x-1)/(1+x) -> 0 + and we can use ATan0() function here + */ + + ValueType n(x),d(x),one,result; + + one.SetOne(); + n.Sub(one); + d.Add(one); + n.Div(d); + + result = ATan0(n); + + n.Set05Pi(); + n.exponent.SubOne(); // =pi/4 + result.Add(n); + + return result; + } + + + /*! + an auxiliary function for calculating the Arc Tangent + where x > 1 + + we're using the formula: + atan(x) = pi/2 - atan(1/x) for x>0 + */ + template + ValueType ATanGreaterThanPlusOne(const ValueType & x) + { + ValueType temp, atan; + + temp.SetOne(); + + if( temp.Div(x) ) + { + // if there was a carry here that means x is very big + // and atan(1/x) fast converged to 0 + atan.SetZero(); + } + else + atan = ATan01(temp); + + temp.Set05Pi(); + temp.Sub(atan); + + return temp; + } + + } // namespace auxiliaryfunctions + + + /*! + this function calculates the Arc Tangent + */ + template + ValueType ATan(ValueType x) + { + using namespace auxiliaryfunctions; + + ValueType one, result; + one.SetOne(); + bool change_sign = false; + + if( x.IsNan() ) + return x; + + // if x is negative we're using the formula: + // atan(-x) = -atan(x) + if( x.IsSign() ) + { + change_sign = true; + x.Abs(); + } + + if( x.GreaterWithoutSignThan(one) ) + result = ATanGreaterThanPlusOne(x); + else + result = ATan01(x); + + if( change_sign ) + result.ChangeSign(); + + return result; + } + + + /*! + this function calculates the Arc Tangent + look at the description of ATan(...) + + (the abbreviation of Arc Tangent can be 'atg' as well) + */ + template + ValueType ATg(const ValueType & x) + { + return ATan(x); + } + + + /*! + this function calculates the Arc Cotangent + + we're using the formula: + actan(x) = pi/2 - atan(x) + */ + template + ValueType ACot(const ValueType & x) + { + ValueType result; + + result.Set05Pi(); + result.Sub(ATan(x)); + + return result; + } + + + /*! + this function calculates the Arc Cotangent + look at the description of ACot(...) + + (the abbreviation of Arc Cotangent can be 'actg' as well) + */ + template + ValueType ACtg(const ValueType & x) + { + return ACot(x); + } + + + /* + * + * hyperbolic functions + * + * + */ + + + /*! + this function calculates the Hyperbolic Sine + + we're using the formula sinh(x)= ( e^x - e^(-x) ) / 2 + */ + template + ValueType Sinh(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType ex, emx; + uint c = 0; + + c += ex.Exp(x); + c += emx.Exp(-x); + + c += ex.Sub(emx); + c += ex.exponent.SubOne(); + + if( err ) + *err = c ? err_overflow : err_ok; + + return ex; + } + + + /*! + this function calculates the Hyperbolic Cosine + + we're using the formula cosh(x)= ( e^x + e^(-x) ) / 2 + */ + template + ValueType Cosh(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType ex, emx; + uint c = 0; + + c += ex.Exp(x); + c += emx.Exp(-x); + + c += ex.Add(emx); + c += ex.exponent.SubOne(); + + if( err ) + *err = c ? err_overflow : err_ok; + + return ex; + } + + + /*! + this function calculates the Hyperbolic Tangent + + we're using the formula tanh(x)= ( e^x - e^(-x) ) / ( e^x + e^(-x) ) + */ + template + ValueType Tanh(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType ex, emx, nominator, denominator; + uint c = 0; + + c += ex.Exp(x); + c += emx.Exp(-x); + + nominator = ex; + c += nominator.Sub(emx); + denominator = ex; + c += denominator.Add(emx); + + c += nominator.Div(denominator); + + if( err ) + *err = c ? err_overflow : err_ok; + + return nominator; + } + + + /*! + this function calculates the Hyperbolic Tangent + look at the description of Tanh(...) + + (the abbreviation of Hyperbolic Tangent can be 'tgh' as well) + */ + template + ValueType Tgh(const ValueType & x, ErrorCode * err = 0) + { + return Tanh(x, err); + } + + /*! + this function calculates the Hyperbolic Cotangent + + we're using the formula coth(x)= ( e^x + e^(-x) ) / ( e^x - e^(-x) ) + */ + template + ValueType Coth(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + if( x.IsZero() ) + { + if( err ) + *err = err_improper_argument; + + return ValueType(); // NaN is set by default + } + + ValueType ex, emx, nominator, denominator; + uint c = 0; + + c += ex.Exp(x); + c += emx.Exp(-x); + + nominator = ex; + c += nominator.Add(emx); + denominator = ex; + c += denominator.Sub(emx); + + c += nominator.Div(denominator); + + if( err ) + *err = c ? err_overflow : err_ok; + + return nominator; + } + + + /*! + this function calculates the Hyperbolic Cotangent + look at the description of Coth(...) + + (the abbreviation of Hyperbolic Cotangent can be 'ctgh' as well) + */ + template + ValueType Ctgh(const ValueType & x, ErrorCode * err = 0) + { + return Coth(x, err); + } + + + /* + * + * inverse hyperbolic functions + * + * + */ + + + /*! + inverse hyperbolic sine + + asinh(x) = ln( x + sqrt(x^2 + 1) ) + */ + template + ValueType ASinh(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType xx(x), one, result; + uint c = 0; + one.SetOne(); + + c += xx.Mul(x); + c += xx.Add(one); + one.exponent.SubOne(); // one=0.5 + // xx is >= 1 + c += xx.PowFrac(one); // xx=sqrt(xx) + c += xx.Add(x); + c += result.Ln(xx); // xx > 0 + + // here can only be a carry + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + /*! + inverse hyperbolic cosine + + acosh(x) = ln( x + sqrt(x^2 - 1) ) x in <1, infinity) + */ + template + ValueType ACosh(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType xx(x), one, result; + uint c = 0; + one.SetOne(); + + if( x < one ) + { + if( err ) + *err = err_improper_argument; + + return result; // NaN is set by default + } + + c += xx.Mul(x); + c += xx.Sub(one); + // xx is >= 0 + // we can't call a PowFrac when the 'x' is zero + // if x is 0 the sqrt(0) is 0 + if( !xx.IsZero() ) + { + one.exponent.SubOne(); // one=0.5 + c += xx.PowFrac(one); // xx=sqrt(xx) + } + c += xx.Add(x); + c += result.Ln(xx); // xx >= 1 + + // here can only be a carry + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + /*! + inverse hyperbolic tangent + + atanh(x) = 0.5 * ln( (1+x) / (1-x) ) x in (-1, 1) + */ + template + ValueType ATanh(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType nominator(x), denominator, one, result; + uint c = 0; + one.SetOne(); + + if( !x.SmallerWithoutSignThan(one) ) + { + if( err ) + *err = err_improper_argument; + + return result; // NaN is set by default + } + + c += nominator.Add(one); + denominator = one; + c += denominator.Sub(x); + c += nominator.Div(denominator); + c += result.Ln(nominator); + c += result.exponent.SubOne(); + + // here can only be a carry + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + /*! + inverse hyperbolic tantent + */ + template + ValueType ATgh(const ValueType & x, ErrorCode * err = 0) + { + return ATanh(x, err); + } + + + /*! + inverse hyperbolic cotangent + + acoth(x) = 0.5 * ln( (x+1) / (x-1) ) x in (-infinity, -1) or (1, infinity) + */ + template + ValueType ACoth(const ValueType & x, ErrorCode * err = 0) + { + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; // NaN + } + + ValueType nominator(x), denominator(x), one, result; + uint c = 0; + one.SetOne(); + + if( !x.GreaterWithoutSignThan(one) ) + { + if( err ) + *err = err_improper_argument; + + return result; // NaN is set by default + } + + c += nominator.Add(one); + c += denominator.Sub(one); + c += nominator.Div(denominator); + c += result.Ln(nominator); + c += result.exponent.SubOne(); + + // here can only be a carry + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + /*! + inverse hyperbolic cotantent + */ + template + ValueType ACtgh(const ValueType & x, ErrorCode * err = 0) + { + return ACoth(x, err); + } + + + + + + /* + * + * functions for converting between degrees, radians and gradians + * + * + */ + + + /*! + this function converts degrees to radians + + it returns: x * pi / 180 + */ + template + ValueType DegToRad(const ValueType & x, ErrorCode * err = 0) + { + ValueType result, temp; + uint c = 0; + + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; + } + + result = x; + + // it is better to make division first and then multiplication + // the result is more accurate especially when x is: 90,180,270 or 360 + temp = 180; + c += result.Div(temp); + + temp.SetPi(); + c += result.Mul(temp); + + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + /*! + this function converts radians to degrees + + it returns: x * 180 / pi + */ + template + ValueType RadToDeg(const ValueType & x, ErrorCode * err = 0) + { + ValueType result, delimiter; + uint c = 0; + + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; + } + + result = 180; + c += result.Mul(x); + + delimiter.SetPi(); + c += result.Div(delimiter); + + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + /*! + this function converts degrees in the long format into one value + + long format: (degrees, minutes, seconds) + minutes and seconds must be greater than or equal zero + + result: + if d>=0 : result= d + ((s/60)+m)/60 + if d<0 : result= d - ((s/60)+m)/60 + + ((s/60)+m)/60 = (s+60*m)/3600 (second version is faster because + there's only one division) + + for example: + DegToDeg(10, 30, 0) = 10.5 + DegToDeg(10, 24, 35.6)=10.4098(8) + */ + template + ValueType DegToDeg( const ValueType & d, const ValueType & m, const ValueType & s, + ErrorCode * err = 0) + { + ValueType delimiter, multipler; + uint c = 0; + + if( d.IsNan() || m.IsNan() || s.IsNan() || m.IsSign() || s.IsSign() ) + { + if( err ) + *err = err_improper_argument; + + delimiter.SetZeroNan(); // not needed, only to get rid of GCC warning about an uninitialized variable + + return delimiter; + } + + multipler = 60; + delimiter = 3600; + + c += multipler.Mul(m); + c += multipler.Add(s); + c += multipler.Div(delimiter); + + if( d.IsSign() ) + multipler.ChangeSign(); + + c += multipler.Add(d); + + if( err ) + *err = c ? err_overflow : err_ok; + + return multipler; + } + + + /*! + this function converts degrees in the long format to radians + */ + template + ValueType DegToRad( const ValueType & d, const ValueType & m, const ValueType & s, + ErrorCode * err = 0) + { + ValueType temp_deg = DegToDeg(d,m,s,err); + + if( err && *err!=err_ok ) + return temp_deg; + + return DegToRad(temp_deg, err); + } + + + /*! + this function converts gradians to radians + + it returns: x * pi / 200 + */ + template + ValueType GradToRad(const ValueType & x, ErrorCode * err = 0) + { + ValueType result, temp; + uint c = 0; + + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; + } + + result = x; + + // it is better to make division first and then multiplication + // the result is more accurate especially when x is: 100,200,300 or 400 + temp = 200; + c += result.Div(temp); + + temp.SetPi(); + c += result.Mul(temp); + + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + /*! + this function converts radians to gradians + + it returns: x * 200 / pi + */ + template + ValueType RadToGrad(const ValueType & x, ErrorCode * err = 0) + { + ValueType result, delimiter; + uint c = 0; + + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; + } + + result = 200; + c += result.Mul(x); + + delimiter.SetPi(); + c += result.Div(delimiter); + + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + /*! + this function converts degrees to gradians + + it returns: x * 200 / 180 + */ + template + ValueType DegToGrad(const ValueType & x, ErrorCode * err = 0) + { + ValueType result, temp; + uint c = 0; + + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; + } + + result = x; + + temp = 200; + c += result.Mul(temp); + + temp = 180; + c += result.Div(temp); + + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + /*! + this function converts degrees in the long format to gradians + */ + template + ValueType DegToGrad( const ValueType & d, const ValueType & m, const ValueType & s, + ErrorCode * err = 0) + { + ValueType temp_deg = DegToDeg(d,m,s,err); + + if( err && *err!=err_ok ) + return temp_deg; + + return DegToGrad(temp_deg, err); + } + + + /*! + this function converts degrees to gradians + + it returns: x * 180 / 200 + */ + template + ValueType GradToDeg(const ValueType & x, ErrorCode * err = 0) + { + ValueType result, temp; + uint c = 0; + + if( x.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return x; + } + + result = x; + + temp = 180; + c += result.Mul(temp); + + temp = 200; + c += result.Div(temp); + + if( err ) + *err = c ? err_overflow : err_ok; + + return result; + } + + + + + /* + * + * another functions + * + * + */ + + + /*! + this function calculates the square root + + Sqrt(9) = 3 + */ + template + ValueType Sqrt(ValueType x, ErrorCode * err = 0) + { + if( x.IsNan() || x.IsSign() ) + { + if( err ) + *err = err_improper_argument; + + x.SetNan(); + + return x; + } + + uint c = x.Sqrt(); + + if( err ) + *err = c ? err_overflow : err_ok; + + return x; + } + + + + namespace auxiliaryfunctions + { + + template + bool RootCheckIndexSign(ValueType & x, const ValueType & index, ErrorCode * err) + { + if( index.IsSign() ) + { + // index cannot be negative + if( err ) + *err = err_improper_argument; + + x.SetNan(); + + return true; + } + + return false; + } + + + template + bool RootCheckIndexZero(ValueType & x, const ValueType & index, ErrorCode * err) + { + if( index.IsZero() ) + { + if( x.IsZero() ) + { + // there isn't root(0;0) - we assume it's not defined + if( err ) + *err = err_improper_argument; + + x.SetNan(); + + return true; + } + + // root(x;0) is 1 (if x!=0) + x.SetOne(); + + if( err ) + *err = err_ok; + + return true; + } + + return false; + } + + + template + bool RootCheckIndexOne(const ValueType & index, ErrorCode * err) + { + ValueType one; + one.SetOne(); + + if( index == one ) + { + //root(x;1) is x + // we do it because if we used the PowFrac function + // we would lose the precision + if( err ) + *err = err_ok; + + return true; + } + + return false; + } + + + template + bool RootCheckIndexTwo(ValueType & x, const ValueType & index, ErrorCode * err) + { + if( index == 2 ) + { + x = Sqrt(x, err); + + return true; + } + + return false; + } + + + template + bool RootCheckIndexFrac(ValueType & x, const ValueType & index, ErrorCode * err) + { + if( !index.IsInteger() ) + { + // index must be integer + if( err ) + *err = err_improper_argument; + + x.SetNan(); + + return true; + } + + return false; + } + + + template + bool RootCheckXZero(ValueType & x, ErrorCode * err) + { + if( x.IsZero() ) + { + // root(0;index) is zero (if index!=0) + // RootCheckIndexZero() must be called beforehand + x.SetZero(); + + if( err ) + *err = err_ok; + + return true; + } + + return false; + } + + + template + bool RootCheckIndex(ValueType & x, const ValueType & index, ErrorCode * err, bool * change_sign) + { + *change_sign = false; + + if( index.Mod2() ) + { + // index is odd (1,3,5...) + if( x.IsSign() ) + { + *change_sign = true; + x.Abs(); + } + } + else + { + // index is even + // x cannot be negative + if( x.IsSign() ) + { + if( err ) + *err = err_improper_argument; + + x.SetNan(); + + return true; + } + } + + return false; + } + + + template + uint RootCorrectInteger(ValueType & old_x, ValueType & x, const ValueType & index) + { + if( !old_x.IsInteger() || x.IsInteger() || !index.exponent.IsSign() ) + return 0; + + // old_x is integer, + // x is not integer, + // index is relatively small (index.exponent<0 or index.exponent<=0) + // (because we're using a special powering algorithm Big::PowUInt()) + + uint c = 0; + + ValueType temp(x); + c += temp.Round(); + + ValueType temp_round(temp); + c += temp.PowUInt(index); + + if( temp == old_x ) + x = temp_round; + + return (c==0)? 0 : 1; + } + + + + } // namespace auxiliaryfunctions + + + + /*! + indexth Root of x + index must be integer and not negative <0;1;2;3....) + + if index==0 the result is one + if x==0 the result is zero and we assume root(0;0) is not defined + + if index is even (2;4;6...) the result is x^(1/index) and x>0 + if index is odd (1;2;3;...) the result is either + -(abs(x)^(1/index)) if x<0 or + x^(1/index)) if x>0 + + (for index==1 the result is equal x) + */ + template + ValueType Root(ValueType x, const ValueType & index, ErrorCode * err = 0) + { + using namespace auxiliaryfunctions; + + if( x.IsNan() || index.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + x.SetNan(); + + return x; + } + + if( RootCheckIndexSign(x, index, err) ) return x; + if( RootCheckIndexZero(x, index, err) ) return x; + if( RootCheckIndexOne ( index, err) ) return x; + if( RootCheckIndexTwo (x, index, err) ) return x; + if( RootCheckIndexFrac(x, index, err) ) return x; + if( RootCheckXZero (x, err) ) return x; + + // index integer and index!=0 + // x!=0 + + ValueType old_x(x); + bool change_sign; + + if( RootCheckIndex(x, index, err, &change_sign ) ) return x; + + ValueType temp; + uint c = 0; + + // we're using the formula: root(x ; n) = exp( ln(x) / n ) + c += temp.Ln(x); + c += temp.Div(index); + c += x.Exp(temp); + + if( change_sign ) + { + // x is different from zero + x.SetSign(); + } + + c += RootCorrectInteger(old_x, x, index); + + if( err ) + *err = c ? err_overflow : err_ok; + + return x; + } + + + + /*! + absolute value of x + e.g. -2 = 2 + 2 = 2 + */ + template + ValueType Abs(const ValueType & x) + { + ValueType result( x ); + result.Abs(); + + return result; + } + + + /*! + it returns the sign of the value + e.g. -2 = -1 + 0 = 0 + 10 = 1 + */ + template + ValueType Sgn(ValueType x) + { + x.Sgn(); + + return x; + } + + + /*! + the remainder from a division + + e.g. + mod( 12.6 ; 3) = 0.6 because 12.6 = 3*4 + 0.6 + mod(-12.6 ; 3) = -0.6 bacause -12.6 = 3*(-4) + (-0.6) + mod( 12.6 ; -3) = 0.6 + mod(-12.6 ; -3) = -0.6 + */ + template + ValueType Mod(ValueType a, const ValueType & b, ErrorCode * err = 0) + { + if( a.IsNan() || b.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + a.SetNan(); + + return a; + } + + uint c = a.Mod(b); + + if( err ) + *err = c ? err_overflow : err_ok; + + return a; + } + + + + namespace auxiliaryfunctions + { + + /*! + this function is used to store factorials in a given container + 'more' means how many values should be added at the end + + e.g. + std::vector fact; + SetFactorialSequence(fact, 3); + // now the container has three values: 1 1 2 + + SetFactorialSequence(fact, 2); + // now the container has five values: 1 1 2 6 24 + */ + template + void SetFactorialSequence(std::vector & fact, uint more = 20) + { + if( more == 0 ) + more = 1; + + uint start = static_cast(fact.size()); + fact.resize(fact.size() + more); + + if( start == 0 ) + { + fact[0] = 1; + ++start; + } + + for(uint i=start ; i + ValueType SetBernoulliNumbersSum(CGamma & cgamma, const ValueType & n_, uint m, + const volatile StopCalculating * stop = 0) + { + ValueType k_, temp, temp2, temp3, sum; + + sum.SetZero(); + + for(uint k=0 ; kWasStopSignal() ) + return ValueType(); // NaN + + if( k>1 && (k & 1) == 1 ) // for that k the Bernoulli number is zero + continue; + + k_ = k; + + temp = n_; // n_ is equal 2 + temp.Pow(k_); + // temp = 2^k + + temp2 = cgamma.fact[m]; + temp3 = cgamma.fact[k]; + temp3.Mul(cgamma.fact[m-k]); + temp2.Div(temp3); + // temp2 = (m k) = m! / ( k! * (m-k)! ) + + temp.Mul(temp2); + temp.Mul(cgamma.bern[k]); + + sum.Add(temp); + // sum += 2^k * (m k) * B(k) + + if( sum.IsNan() ) + break; + } + + return sum; + } + + + /*! + an auxiliary function used to calculate Bernoulli numbers + start is >= 2 + + we use the recurrence formula: + B(m) = 1 / (2*(1 - 2^m)) * sum(m) + where sum(m) is calculated by SetBernoulliNumbersSum() + */ + template + bool SetBernoulliNumbersMore(CGamma & cgamma, uint start, const volatile StopCalculating * stop = 0) + { + ValueType denominator, temp, temp2, temp3, m_, sum, sum2, n_, k_; + + const uint n = 2; + n_ = n; + + // start is >= 2 + for(uint m=start ; mWasStopSignal() ) + { + cgamma.bern.resize(m); // valid numbers are in [0, m-1] + return false; + } + + cgamma.bern[m].Div(denominator); + } + } + + return true; + } + + + /*! + this function is used to calculate Bernoulli numbers, + returns false if there was a stop signal, + 'more' means how many values should be added at the end + + e.g. + typedef Big<1,2> MyBig; + CGamma cgamma; + SetBernoulliNumbers(cgamma, 3); + // now we have three first Bernoulli numbers: 1 -0.5 0.16667 + + SetBernoulliNumbers(cgamma, 4); + // now we have 7 Bernoulli numbers: 1 -0.5 0.16667 0 -0.0333 0 0.0238 + */ + template + bool SetBernoulliNumbers(CGamma & cgamma, uint more = 20, const volatile StopCalculating * stop = 0) + { + if( more == 0 ) + more = 1; + + uint start = static_cast(cgamma.bern.size()); + cgamma.bern.resize(cgamma.bern.size() + more); + + if( start == 0 ) + { + cgamma.bern[0].SetOne(); + ++start; + } + + if( cgamma.bern.size() == 1 ) + return true; + + if( start == 1 ) + { + cgamma.bern[1].Set05(); + cgamma.bern[1].ChangeSign(); + ++start; + } + + // we should have sufficient factorials in cgamma.fact + if( cgamma.fact.size() < cgamma.bern.size() ) + SetFactorialSequence(cgamma.fact, static_cast(cgamma.bern.size() - cgamma.fact.size())); + + + return SetBernoulliNumbersMore(cgamma, start, stop); + } + + + /*! + an auxiliary function used to calculate the Gamma() function + + we calculate a sum: + sum(n) = sum_{m=2} { B(m) / ( (m^2 - m) * n^(m-1) ) } = 1/(12*n) - 1/(360*n^3) + 1/(1260*n^5) + ... + B(m) means a mth Bernoulli number + the sum starts from m=2, we calculate as long as the value will not change after adding a next part + */ + template + ValueType GammaFactorialHighSum(const ValueType & n, CGamma & cgamma, ErrorCode & err, + const volatile StopCalculating * stop) + { + ValueType temp, temp2, denominator, sum, oldsum; + + sum.SetZero(); + + for(uint m=2 ; mWasStopSignal() ) + { + err = err_interrupt; + return ValueType(); // NaN + } + + temp = (m-1); + denominator = n; + denominator.Pow(temp); + // denominator = n ^ (m-1) + + temp = m; + temp2 = temp; + temp.Mul(temp2); + temp.Sub(temp2); + // temp = m^2 - m + + denominator.Mul(temp); + // denominator = (m^2 - m) * n ^ (m-1) + + if( m >= cgamma.bern.size() ) + { + if( !SetBernoulliNumbers(cgamma, m - cgamma.bern.size() + 1 + 3, stop) ) // 3 more than needed + { + // there was the stop signal + err = err_interrupt; + return ValueType(); // NaN + } + } + + temp = cgamma.bern[m]; + temp.Div(denominator); + + oldsum = sum; + sum.Add(temp); + + if( sum.IsNan() || oldsum==sum ) + break; + } + + return sum; + } + + + /*! + an auxiliary function used to calculate the Gamma() function + + we calculate a helper function GammaFactorialHigh() by using Stirling's series: + n! = (n/e)^n * sqrt(2*pi*n) * exp( sum(n) ) + where n is a real number (not only an integer) and is sufficient large (greater than TTMATH_GAMMA_BOUNDARY) + and sum(n) is calculated by GammaFactorialHighSum() + */ + template + ValueType GammaFactorialHigh(const ValueType & n, CGamma & cgamma, ErrorCode & err, + const volatile StopCalculating * stop) + { + ValueType temp, temp2, temp3, denominator, sum; + + temp.Set2Pi(); + temp.Mul(n); + temp2 = Sqrt(temp); + // temp2 = sqrt(2*pi*n) + + temp = n; + temp3.SetE(); + temp.Div(temp3); + temp.Pow(n); + // temp = (n/e)^n + + sum = GammaFactorialHighSum(n, cgamma, err, stop); + temp3.Exp(sum); + // temp3 = exp(sum) + + temp.Mul(temp2); + temp.Mul(temp3); + + return temp; + } + + + /*! + an auxiliary function used to calculate the Gamma() function + + Gamma(x) = GammaFactorialHigh(x-1) + */ + template + ValueType GammaPlusHigh(ValueType n, CGamma & cgamma, ErrorCode & err, const volatile StopCalculating * stop) + { + ValueType one; + + one.SetOne(); + n.Sub(one); + + return GammaFactorialHigh(n, cgamma, err, stop); + } + + + /*! + an auxiliary function used to calculate the Gamma() function + + we use this function when n is integer and a small value (from 0 to TTMATH_GAMMA_BOUNDARY] + we use the formula: + gamma(n) = (n-1)! = 1 * 2 * 3 * ... * (n-1) + */ + template + ValueType GammaPlusLowIntegerInt(uint n, CGamma & cgamma) + { + TTMATH_ASSERT( n > 0 ) + + if( n - 1 < static_cast(cgamma.fact.size()) ) + return cgamma.fact[n - 1]; + + ValueType res; + uint start = 2; + + if( cgamma.fact.size() < 2 ) + { + res.SetOne(); + } + else + { + start = static_cast(cgamma.fact.size()); + res = cgamma.fact[start-1]; + } + + for(uint i=start ; i + ValueType GammaPlusLowInteger(const ValueType & n, CGamma & cgamma) + { + sint n_; + + n.ToInt(n_); + + return GammaPlusLowIntegerInt(n_, cgamma); + } + + + /*! + an auxiliary function used to calculate the Gamma() function + + we use this function when n is a small value (from 0 to TTMATH_GAMMA_BOUNDARY] + we use a recurrence formula: + gamma(z+1) = z * gamma(z) + then: gamma(z) = gamma(z+1) / z + + e.g. + gamma(3.89) = gamma(2001.89) / ( 3.89 * 4.89 * 5.89 * ... * 1999.89 * 2000.89 ) + */ + template + ValueType GammaPlusLow(ValueType n, CGamma & cgamma, ErrorCode & err, const volatile StopCalculating * stop) + { + ValueType one, denominator, temp, boundary; + + if( n.IsInteger() ) + return GammaPlusLowInteger(n, cgamma); + + one.SetOne(); + denominator = n; + boundary = TTMATH_GAMMA_BOUNDARY; + + while( n < boundary ) + { + n.Add(one); + denominator.Mul(n); + } + + n.Add(one); + + // now n is sufficient big + temp = GammaPlusHigh(n, cgamma, err, stop); + temp.Div(denominator); + + return temp; + } + + + /*! + an auxiliary function used to calculate the Gamma() function + */ + template + ValueType GammaPlus(const ValueType & n, CGamma & cgamma, ErrorCode & err, const volatile StopCalculating * stop) + { + if( n > TTMATH_GAMMA_BOUNDARY ) + return GammaPlusHigh(n, cgamma, err, stop); + + return GammaPlusLow(n, cgamma, err, stop); + } + + + /*! + an auxiliary function used to calculate the Gamma() function + + this function is used when n is negative + we use the reflection formula: + gamma(1-z) * gamma(z) = pi / sin(pi*z) + then: gamma(z) = pi / (sin(pi*z) * gamma(1-z)) + + */ + template + ValueType GammaMinus(const ValueType & n, CGamma & cgamma, ErrorCode & err, const volatile StopCalculating * stop) + { + ValueType pi, denominator, temp, temp2; + + if( n.IsInteger() ) + { + // gamma function is not defined when n is negative and integer + err = err_improper_argument; + return temp; // NaN + } + + pi.SetPi(); + + temp = pi; + temp.Mul(n); + temp2 = Sin(temp); + // temp2 = sin(pi * n) + + temp.SetOne(); + temp.Sub(n); + temp = GammaPlus(temp, cgamma, err, stop); + // temp = gamma(1 - n) + + temp.Mul(temp2); + pi.Div(temp); + + return pi; + } + + } // namespace auxiliaryfunctions + + + + /*! + this function calculates the Gamma function + + it's multithread safe, you should create a CGamma<> object and use it whenever you call the Gamma() + e.g. + typedef Big<1,2> MyBig; + MyBig x=234, y=345.53; + CGamma cgamma; + std::cout << Gamma(x, cgamma) << std::endl; + std::cout << Gamma(y, cgamma) << std::endl; + in the CGamma<> object the function stores some coefficients (factorials, Bernoulli numbers), + and they will be reused in next calls to the function + + each thread should have its own CGamma<> object, and you can use these objects with Factorial() function too + */ + template + ValueType Gamma(const ValueType & n, CGamma & cgamma, ErrorCode * err = 0, + const volatile StopCalculating * stop = 0) + { + using namespace auxiliaryfunctions; + + ValueType result; + ErrorCode err_tmp; + + if( n.IsNan() ) + { + if( err ) + *err = err_improper_argument; + + return n; + } + + if( cgamma.history.Get(n, result, err_tmp) ) + { + if( err ) + *err = err_tmp; + + return result; + } + + err_tmp = err_ok; + + if( n.IsSign() ) + { + result = GammaMinus(n, cgamma, err_tmp, stop); + } + else + if( n.IsZero() ) + { + err_tmp = err_improper_argument; + result.SetNan(); + } + else + { + result = GammaPlus(n, cgamma, err_tmp, stop); + } + + if( result.IsNan() && err_tmp==err_ok ) + err_tmp = err_overflow; + + if( err ) + *err = err_tmp; + + if( stop && !stop->WasStopSignal() ) + cgamma.history.Add(n, result, err_tmp); + + return result; + } + + + /*! + this function calculates the Gamma function + + note: this function should be used only in a single-thread environment + */ + template + ValueType Gamma(const ValueType & n, ErrorCode * err = 0) + { + // warning: this static object is not thread safe + static CGamma cgamma; + + return Gamma(n, cgamma, err); + } + + + + namespace auxiliaryfunctions + { + + /*! + an auxiliary function for calculating the factorial function + + we use the formula: + x! = gamma(x+1) + */ + template + ValueType Factorial2(ValueType x, + CGamma * cgamma = 0, + ErrorCode * err = 0, + const volatile StopCalculating * stop = 0) + { + ValueType result, one; + + if( x.IsNan() || x.IsSign() || !x.IsInteger() ) + { + if( err ) + *err = err_improper_argument; + + x.SetNan(); + + return x; + } + + one.SetOne(); + x.Add(one); + + if( cgamma ) + return Gamma(x, *cgamma, err, stop); + + return Gamma(x, err); + } + + } // namespace auxiliaryfunctions + + + + /*! + the factorial from given 'x' + e.g. + Factorial(4) = 4! = 1*2*3*4 + + it's multithread safe, you should create a CGamma<> object and use it whenever you call the Factorial() + e.g. + typedef Big<1,2> MyBig; + MyBig x=234, y=54345; + CGamma cgamma; + std::cout << Factorial(x, cgamma) << std::endl; + std::cout << Factorial(y, cgamma) << std::endl; + in the CGamma<> object the function stores some coefficients (factorials, Bernoulli numbers), + and they will be reused in next calls to the function + + each thread should have its own CGamma<> object, and you can use these objects with Gamma() function too + */ + template + ValueType Factorial(const ValueType & x, CGamma & cgamma, ErrorCode * err = 0, + const volatile StopCalculating * stop = 0) + { + return auxiliaryfunctions::Factorial2(x, &cgamma, err, stop); + } + + + /*! + the factorial from given 'x' + e.g. + Factorial(4) = 4! = 1*2*3*4 + + note: this function should be used only in a single-thread environment + */ + template + ValueType Factorial(const ValueType & x, ErrorCode * err = 0) + { + return auxiliaryfunctions::Factorial2(x, (CGamma*)0, err, 0); + } + + + /*! + this method prepares some coefficients: factorials and Bernoulli numbers + stored in 'fact' and 'bern' objects + + we're defining the method here because we're using Gamma() function which + is not available in ttmathobjects.h + + read the doc info in ttmathobjects.h file where CGamma<> struct is declared + */ + template + void CGamma::InitAll() + { + ValueType x = TTMATH_GAMMA_BOUNDARY + 1; + + // history.Remove(x) removes only one object + // we must be sure that there are not others objects with the key 'x' + while( history.Remove(x) ) + { + } + + // the simplest way to initialize is to call the Gamma function with (TTMATH_GAMMA_BOUNDARY + 1) + // when x is larger then fewer coefficients we need + Gamma(x, *this); + } + + + +} // namespace + + +#ifdef _MSC_VER +//warning C4127: conditional expression is constant +#pragma warning( default: 4127 ) +//warning C4702: unreachable code +#pragma warning( default: 4702 ) +//warning C4800: forcing value to bool 'true' or 'false' (performance warning) +#pragma warning( default: 4800 ) +#endif + +#endif diff --git a/extern/ttmath/ttmathint.h b/extern/ttmath/ttmathint.h new file mode 100644 index 0000000000..8ad0189f93 --- /dev/null +++ b/extern/ttmath/ttmathint.h @@ -0,0 +1,1917 @@ +/* + * This file is a part of TTMath Bignum Library + * and is distributed under the (new) BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2006-2011, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name Tomasz Sowa nor the names of contributors to this + * project may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + + + +#ifndef headerfilettmathint +#define headerfilettmathint + +/*! + \file ttmathint.h + \brief template class Int +*/ + +#include "ttmathuint.h" + +namespace ttmath +{ + + +/*! + \brief Int implements a big integer value with a sign + + value_size - how many bytes specify our value + on 32bit platforms: value_size=1 -> 4 bytes -> 32 bits + on 64bit platforms: value_size=1 -> 8 bytes -> 64 bits + value_size = 1,2,3,4,5,6.... +*/ +template +class Int : public UInt +{ +public: + + /*! + this method sets the max value which this class can hold + (all bits will be one besides the last one) + */ + void SetMax() + { + UInt::SetMax(); + UInt::table[value_size-1] = ~ TTMATH_UINT_HIGHEST_BIT; + } + + + /*! + this method sets the min value which this class can hold + (all bits will be zero besides the last one which is one) + */ + void SetMin() + { + UInt::SetZero(); + UInt::table[value_size-1] = TTMATH_UINT_HIGHEST_BIT; + } + + + /*! + this method sets -1 as the value + (-1 is equal the max value in an unsigned type) + */ + void SetSignOne() + { + UInt::SetMax(); + } + + + /*! + we change the sign of the value + + if it isn't possible to change the sign this method returns 1 + else return 0 and changing the sign + */ + uint ChangeSign() + { + /* + if the value is equal that one which has been returned from SetMin + (only the highest bit is set) that means we can't change sign + because the value is too big (bigger about one) + + e.g. when value_size = 1 and value is -2147483648 we can't change it to the + 2147483648 because the max value which can be held is 2147483647 + + we don't change the value and we're using this fact somewhere in some methods + (if we look on our value without the sign we get the correct value + eg. -2147483648 in Int<1> will be 2147483648 on the UInt<1> type) + */ + if( UInt::IsOnlyTheHighestBitSet() ) + return 1; + + UInt temp(*this); + UInt::SetZero(); + UInt::Sub(temp); + + return 0; + } + + + + /*! + this method sets the sign + + e.g. 1 -> -1 + -2 -> -2 + + from a positive value we make a negative value, + if the value is negative we do nothing + */ + void SetSign() + { + if( IsSign() ) + return; + + ChangeSign(); + } + + + + /*! + this method returns true if there's the sign + + (the highest bit will be converted to the bool) + */ + bool IsSign() const + { + return UInt::IsTheHighestBitSet(); + } + + + + /*! + it sets an absolute value + + it can return carry (1) (look on ChangeSign() for details) + */ + uint Abs() + { + if( !IsSign() ) + return 0; + + return ChangeSign(); + } + + + + + /*! + * + * basic mathematic functions + * + */ + +private: + + uint CorrectCarryAfterAdding(bool p1_is_sign, bool p2_is_sign) + { + if( !p1_is_sign && !p2_is_sign ) + { + if( UInt::IsTheHighestBitSet() ) + return 1; + } + + if( p1_is_sign && p2_is_sign ) + { + if( ! UInt::IsTheHighestBitSet() ) + return 1; + } + + return 0; + } + + +public: + + /*! + this method adds two value with a sign and returns a carry + + we're using methods from the base class because values are stored with U2 + we must only make the carry correction + + this = p1(=this) + p2 + + when p1>=0 i p2>=0 carry is set when the highest bit of value is set + when p1<0 i p2<0 carry is set when the highest bit of value is clear + when p1>=0 i p2<0 carry will never be set + when p1<0 i p2>=0 carry will never be set + */ + uint Add(const Int & ss2) + { + bool p1_is_sign = IsSign(); + bool p2_is_sign = ss2.IsSign(); + + UInt::Add(ss2); + + return CorrectCarryAfterAdding(p1_is_sign, p2_is_sign); + } + + + /*! + this method adds one *unsigned* word (at a specific position) + and returns a carry (if it was) + + look at a description in UInt<>::AddInt(...) + */ + uint AddInt(uint value, uint index = 0) + { + bool p1_is_sign = IsSign(); + + UInt::AddInt(value, index); + + return CorrectCarryAfterAdding(p1_is_sign, false); + } + + + /*! + this method adds two *unsigned* words to the existing value + and these words begin on the 'index' position + + index should be equal or smaller than value_size-2 (index <= value_size-2) + x1 - lower word, x2 - higher word + + look at a description in UInt<>::AddTwoInts(...) + */ + uint AddTwoInts(uint x2, uint x1, uint index) + { + bool p1_is_sign = IsSign(); + + UInt::AddTwoInts(x2, x1, index); + + return CorrectCarryAfterAdding(p1_is_sign, false); + } + +private: + + uint CorrectCarryAfterSubtracting(bool p1_is_sign, bool p2_is_sign) + { + if( !p1_is_sign && p2_is_sign ) + { + if( UInt::IsTheHighestBitSet() ) + return 1; + } + + if( p1_is_sign && !p2_is_sign ) + { + if( ! UInt::IsTheHighestBitSet() ) + return 1; + } + + return 0; + } + +public: + + /*! + this method subtracts two values with a sign + + we don't use the previous Add because the method ChangeSign can + sometimes return carry + + this = p1(=this) - p2 + + when p1>=0 i p2>=0 carry will never be set + when p1<0 i p2<0 carry will never be set + when p1>=0 i p2<0 carry is set when the highest bit of value is set + when p1<0 i p2>=0 carry is set when the highest bit of value is clear + */ + uint Sub(const Int & ss2) + { + bool p1_is_sign = IsSign(); + bool p2_is_sign = ss2.IsSign(); + + UInt::Sub(ss2); + + return CorrectCarryAfterSubtracting(p1_is_sign, p2_is_sign); + } + + + /*! + this method subtracts one *unsigned* word (at a specific position) + and returns a carry (if it was) + */ + uint SubInt(uint value, uint index = 0) + { + bool p1_is_sign = IsSign(); + + UInt::SubInt(value, index); + + return CorrectCarryAfterSubtracting(p1_is_sign, false); + } + + + /*! + this method adds one to the value and returns carry + */ + uint AddOne() + { + bool p1_is_sign = IsSign(); + + UInt::AddOne(); + + return CorrectCarryAfterAdding(p1_is_sign, false); + } + + + /*! + this method subtracts one from the value and returns carry + */ + uint SubOne() + { + bool p1_is_sign = IsSign(); + + UInt::SubOne(); + + return CorrectCarryAfterSubtracting(p1_is_sign, false); + } + + +private: + + + uint CheckMinCarry(bool ss1_is_sign, bool ss2_is_sign) + { + /* + we have to examine the sign of the result now + but if the result is with the sign then: + 1. if the signs were the same that means the result is too big + (the result must be without a sign) + 2. if the signs were different that means if the result + is different from that one which has been returned from SetMin() + that is carry (result too big) but if the result is equal SetMin() + there'll be ok (and the next SetSign will has no effect because + the value is actually negative -- look at description of that case + in ChangeSign()) + */ + if( IsSign() ) + { + if( ss1_is_sign != ss2_is_sign ) + { + /* + there can be one case where signs are different and + the result will be equal the value from SetMin() (only the highest bit is set) + (this situation is ok) + */ + if( !UInt::IsOnlyTheHighestBitSet() ) + return 1; + } + else + { + // signs were the same + return 1; + } + } + + return 0; + } + + +public: + + + /*! + multiplication: this = this * ss2 + + it can return a carry + */ + uint MulInt(sint ss2) + { + bool ss1_is_sign, ss2_is_sign; + uint c; + + ss1_is_sign = IsSign(); + + /* + we don't have to check the carry from Abs (values will be correct + because next we're using the method MulInt from the base class UInt + which is without a sign) + */ + Abs(); + + if( ss2 < 0 ) + { + ss2 = -ss2; + ss2_is_sign = true; + } + else + { + ss2_is_sign = false; + } + + c = UInt::MulInt((uint)ss2); + c += CheckMinCarry(ss1_is_sign, ss2_is_sign); + + if( ss1_is_sign != ss2_is_sign ) + SetSign(); + + return c; + } + + + + /*! + multiplication this = this * ss2 + + it returns carry if the result is too big + (we're using the method from the base class but we have to make + one correction in account of signs) + */ + uint Mul(Int ss2) + { + bool ss1_is_sign, ss2_is_sign; + uint c; + + ss1_is_sign = IsSign(); + ss2_is_sign = ss2.IsSign(); + + /* + we don't have to check the carry from Abs (values will be correct + because next we're using the method Mul from the base class UInt + which is without a sign) + */ + Abs(); + ss2.Abs(); + + c = UInt::Mul(ss2); + c += CheckMinCarry(ss1_is_sign, ss2_is_sign); + + if( ss1_is_sign != ss2_is_sign ) + SetSign(); + + return c; + } + + + /*! + division this = this / ss2 + returned values: + 0 - ok + 1 - division by zero + + for example: (result means 'this') + 20 / 3 --> result: 6 remainder: 2 + -20 / 3 --> result: -6 remainder: -2 + 20 / -3 --> result: -6 remainder: 2 + -20 / -3 --> result: 6 remainder: -2 + + in other words: this(old) = ss2 * this(new)(result) + remainder + */ + uint Div(Int ss2, Int * remainder = 0) + { + bool ss1_is_sign, ss2_is_sign; + + ss1_is_sign = IsSign(); + ss2_is_sign = ss2.IsSign(); + + /* + we don't have to test the carry from Abs as well as in Mul + */ + Abs(); + ss2.Abs(); + + uint c = UInt::Div(ss2, remainder); + + if( ss1_is_sign != ss2_is_sign ) + SetSign(); + + if( ss1_is_sign && remainder ) + remainder->SetSign(); + + return c; + } + + uint Div(const Int & ss2, Int & remainder) + { + return Div(ss2, &remainder); + } + + + /*! + division this = this / ss2 (ss2 is int) + returned values: + 0 - ok + 1 - division by zero + + for example: (result means 'this') + 20 / 3 --> result: 6 remainder: 2 + -20 / 3 --> result: -6 remainder: -2 + 20 / -3 --> result: -6 remainder: 2 + -20 / -3 --> result: 6 remainder: -2 + + in other words: this(old) = ss2 * this(new)(result) + remainder + */ + uint DivInt(sint ss2, sint * remainder = 0) + { + bool ss1_is_sign, ss2_is_sign; + + ss1_is_sign = IsSign(); + + /* + we don't have to test the carry from Abs as well as in Mul + */ + Abs(); + + if( ss2 < 0 ) + { + ss2 = -ss2; + ss2_is_sign = true; + } + else + { + ss2_is_sign = false; + } + + uint rem; + uint c = UInt::DivInt((uint)ss2, &rem); + + if( ss1_is_sign != ss2_is_sign ) + SetSign(); + + if( remainder ) + { + if( ss1_is_sign ) + *remainder = -sint(rem); + else + *remainder = sint(rem); + } + + return c; + } + + + uint DivInt(sint ss2, sint & remainder) + { + return DivInt(ss2, &remainder); + } + + +private: + + + /*! + power this = this ^ pow + this can be negative + pow is >= 0 + */ + uint Pow2(const Int & pow) + { + bool was_sign = IsSign(); + uint c = 0; + + if( was_sign ) + c += Abs(); + + uint c_temp = UInt::Pow(pow); + if( c_temp > 0 ) + return c_temp; // c_temp can be: 0, 1 or 2 + + if( was_sign && (pow.table[0] & 1) == 1 ) + // negative value to the power of odd number is negative + c += ChangeSign(); + + return (c==0)? 0 : 1; + } + + +public: + + + /*! + power this = this ^ pow + + return values: + 0 - ok + 1 - carry + 2 - incorrect arguments 0^0 or 0^(-something) + */ + uint Pow(Int pow) + { + if( !pow.IsSign() ) + return Pow2(pow); + + if( UInt::IsZero() ) + // if 'pow' is negative then + // 'this' must be different from zero + return 2; + + if( pow.ChangeSign() ) + return 1; + + Int t(*this); + uint c_temp = t.Pow2(pow); + if( c_temp > 0 ) + return c_temp; + + UInt::SetOne(); + if( Div(t) ) + return 1; + + return 0; + } + + + + /*! + * + * convertion methods + * + */ +private: + + + /*! + an auxiliary method for converting both from UInt and Int + */ + template + uint FromUIntOrInt(const UInt & p, bool UInt_type) + { + uint min_size = (value_size < argument_size)? value_size : argument_size; + uint i; + + for(i=0 ; i::table[i] = p.table[i]; + + + if( value_size > argument_size ) + { + uint fill; + + if( UInt_type ) + fill = 0; + else + fill = (p.table[argument_size-1] & TTMATH_UINT_HIGHEST_BIT)? + TTMATH_UINT_MAX_VALUE : 0; + + // 'this' is longer than 'p' + for( ; i::table[i] = fill; + } + else + { + uint test = (UInt::table[value_size-1] & TTMATH_UINT_HIGHEST_BIT)? + TTMATH_UINT_MAX_VALUE : 0; + + if( UInt_type && test!=0 ) + return 1; + + for( ; i type into this class + + this operation has mainly sense if the value from p + can be held in this type + + it returns a carry if the value 'p' is too big + */ + template + uint FromInt(const Int & p) + { + return FromUIntOrInt(p, false); + } + + + /*! + this method converts the sint type into this class + */ + uint FromInt(sint value) + { + uint fill = ( value<0 ) ? TTMATH_UINT_MAX_VALUE : 0; + + for(uint i=1 ; i::table[i] = fill; + + UInt::table[0] = uint(value); + + // there'll never be a carry here + return 0; + } + + + /*! + this method converts UInt into this class + */ + template + uint FromUInt(const UInt & p) + { + return FromUIntOrInt(p, true); + } + + + /*! + this method converts UInt into this class + */ + template + uint FromInt(const UInt & p) + { + return FromUIntOrInt(p, true); + } + + + /*! + this method converts the uint type into this class + */ + uint FromUInt(uint value) + { + for(uint i=1 ; i::table[i] = 0; + + UInt::table[0] = value; + + // there can be a carry here when the size of this value is equal one word + // and the 'value' has the highest bit set + if( value_size==1 && (value & TTMATH_UINT_HIGHEST_BIT)!=0 ) + return 1; + + return 0; + } + + + /*! + this method converts the uint type into this class + */ + uint FromInt(uint value) + { + return FromUInt(value); + } + + + /*! + the default assignment operator + */ +/* Int & operator=(const Int & p) + { + FromInt(p); + + return *this; + } + */ + + /*! + this operator converts an Int type to this class + + it doesn't return a carry + */ +/* template + Int & operator=(const Int & p) + { + FromInt(p); + + return *this; + } + */ + + /*! + this method converts the sint type to this class + */ + Int & operator=(sint i) + { + FromInt(i); + + return *this; + } + + + /*! + a constructor for converting the uint to this class + */ +/* Int(sint i) + { + FromInt(i); + } +*/ + + /*! + a copy constructor + */ +/* Int(const Int & u) + { + FromInt(u); + } +*/ + + /*! + a constructor for copying from another types + */ +/* template + Int(const Int & u) + { + // look that 'size' we still set as 'value_size' and not as u.value_size + FromInt(u); + } +*/ + + + /*! + this operator converts an UInt type to this class + + it doesn't return a carry + */ + template + Int & operator=(const UInt & p) + { + FromUInt(p); + + return *this; + } + + + /*! + this method converts the Uint type to this class + */ + Int & operator=(uint i) + { + FromUInt(i); + + return *this; + } + + + /*! + a constructor for converting the uint to this class + */ +/* Int(uint i) + { + FromUInt(i); + } +*/ + + /*! + a constructor for copying from another types + */ +/* template + Int(const UInt & u) + { + // look that 'size' we still set as 'value_size' and not as u.value_size + FromUInt(u); + } +*/ + + +#ifdef TTMATH_PLATFORM32 + + + /*! + this method converts unsigned 64 bit int type to this class + ***this method is created only on a 32bit platform*** + */ + uint FromUInt(ulint n) + { + uint c = UInt::FromUInt(n); + + if( c ) + return 1; + + if( value_size == 1 ) + return ((UInt::table[0] & TTMATH_UINT_HIGHEST_BIT) == 0) ? 0 : 1; + + if( value_size == 2 ) + return ((UInt::table[1] & TTMATH_UINT_HIGHEST_BIT) == 0) ? 0 : 1; + + return 0; + } + + + /*! + this method converts unsigned 64 bit int type to this class + ***this method is created only on a 32bit platform*** + */ + uint FromInt(ulint n) + { + return FromUInt(n); + } + + + /*! + this method converts signed 64 bit int type to this class + ***this method is created only on a 32bit platform*** + */ + uint FromInt(slint n) + { + uint mask = (n < 0) ? TTMATH_UINT_MAX_VALUE : 0; + + UInt::table[0] = (uint)(ulint)n; + + if( value_size == 1 ) + { + if( uint(ulint(n) >> 32) != mask ) + return 1; + + return ((UInt::table[0] & TTMATH_UINT_HIGHEST_BIT) == (mask & TTMATH_UINT_HIGHEST_BIT)) ? 0 : 1; + } + + UInt::table[1] = (uint)(ulint(n) >> 32); + + for(uint i=2 ; i::table[i] = mask; + + return 0; + } + + + /*! + this operator converts unsigned 64 bit int type to this class + ***this operator is created only on a 32bit platform*** + */ + Int & operator=(ulint n) + { + FromUInt(n); + + return *this; + } + + + /*! + a constructor for converting unsigned 64 bit int to this class + ***this constructor is created only on a 32bit platform*** + */ +/* Int(ulint n) + { + FromUInt(n); + } +*/ + + /*! + this operator converts signed 64 bit int type to this class + ***this operator is created only on a 32bit platform*** + */ + Int & operator=(slint n) + { + FromInt(n); + + return *this; + } + + + /*! + a constructor for converting signed 64 bit int to this class + ***this constructor is created only on a 32bit platform*** + */ +/* Int(slint n) + { + FromInt(n); + } +*/ +#endif + + + + +#ifdef TTMATH_PLATFORM64 + + /*! + this method converts 32 bit unsigned int type to this class + ***this operator is created only on a 64bit platform*** + */ + uint FromUInt(unsigned int i) + { + return FromUInt(uint(i)); + } + + + /*! + this method converts 32 bit unsigned int type to this class + ***this operator is created only on a 64bit platform*** + */ + uint FromInt(unsigned int i) + { + return FromUInt(i); + } + + + /*! + this method converts 32 bit signed int type to this class + ***this operator is created only on a 64bit platform*** + */ + uint FromInt(signed int i) + { + return FromInt(sint(i)); + } + + + /*! + this method converts 32 bit unsigned int type to this class + ***this operator is created only on a 64bit platform*** + */ + Int & operator=(unsigned int i) + { + FromUInt(i); + + return *this; + } + + + /*! + a constructor for converting 32 bit unsigned int to this class + ***this constructor is created only on a 64bit platform*** + */ +/* Int(unsigned int i) + { + FromUInt(i); + } +*/ + + /*! + this operator converts 32 bit signed int type to this class + ***this operator is created only on a 64bit platform*** + */ + Int & operator=(signed int i) + { + FromInt(i); + + return *this; + } + + + /*! + a constructor for converting 32 bit signed int to this class + ***this constructor is created only on a 64bit platform*** + */ +/* Int(signed int i) + { + FromInt(i); + } +*/ +#endif + + + + /*! + a constructor for converting string to this class (with the base=10) + */ +/* Int(const char * s) + { + FromString(s); + } +*/ + + /*! + a constructor for converting a string to this class (with the base=10) + */ +/* Int(const std::string & s) + { + FromString( s.c_str() ); + } +*/ + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + a constructor for converting string to this class (with the base=10) + */ + Int(const wchar_t * s) + { + FromString(s); + } + + + /*! + a constructor for converting a string to this class (with the base=10) + */ + Int(const std::wstring & s) + { + FromString( s.c_str() ); + } + +#endif + + + /*! + a default constructor + + we don't clear table etc. + */ +/* Int() + { + } +*/ + + /*! + the destructor + */ +/* ~Int() + { + } +*/ + + /*! + this method returns the lowest value from table with a sign + + we must be sure when we using this method whether the value + will hold in an sint type or not (the rest value from table must be zero or -1) + */ + sint ToInt() const + { + return sint( UInt::table[0] ); + } + + + /*! + this method converts the value to uint type + can return a carry if the value is too long to store it in uint type + */ + uint ToUInt(uint & result) const + { + uint c = UInt::ToUInt(result); + + if( value_size == 1 ) + return (result & TTMATH_UINT_HIGHEST_BIT) == 0 ? 0 : 1; + + return c; + } + + + /*! + this method converts the value to uint type + can return a carry if the value is too long to store it in uint type + */ + uint ToInt(uint & result) const + { + return ToUInt(result); + } + + + /*! + this method converts the value to sint type + can return a carry if the value is too long to store it in sint type + */ + uint ToInt(sint & result) const + { + result = sint( UInt::table[0] ); + uint mask = IsSign() ? TTMATH_UINT_MAX_VALUE : 0; + + if( (result & TTMATH_UINT_HIGHEST_BIT) != (mask & TTMATH_UINT_HIGHEST_BIT) ) + return 1; + + for(uint i=1 ; i::table[i] != mask ) + return 1; + + return 0; + } + + +#ifdef TTMATH_PLATFORM32 + + /*! + this method converts the value to ulint type (64 bit unsigned integer) + can return a carry if the value is too long to store it in ulint type + *** this method is created only on a 32 bit platform *** + */ + uint ToUInt(ulint & result) const + { + uint c = UInt::ToUInt(result); + + if( value_size == 1 ) + return (UInt::table[0] & TTMATH_UINT_HIGHEST_BIT) == 0 ? 0 : 1; + + if( value_size == 2 ) + return (UInt::table[1] & TTMATH_UINT_HIGHEST_BIT) == 0 ? 0 : 1; + + return c; + } + + + /*! + this method converts the value to ulint type (64 bit unsigned integer) + can return a carry if the value is too long to store it in ulint type + *** this method is created only on a 32 bit platform *** + */ + uint ToInt(ulint & result) const + { + return ToUInt(result); + } + + + /*! + this method converts the value to slint type (64 bit signed integer) + can return a carry if the value is too long to store it in slint type + *** this method is created only on a 32 bit platform *** + */ + uint ToInt(slint & result) const + { + if( value_size == 1 ) + { + result = slint(sint(UInt::table[0])); + } + else + { + uint low = UInt::table[0]; + uint high = UInt::table[1]; + + result = low; + result |= (ulint(high) << TTMATH_BITS_PER_UINT); + + uint mask = IsSign() ? TTMATH_UINT_MAX_VALUE : 0; + + if( (high & TTMATH_UINT_HIGHEST_BIT) != (mask & TTMATH_UINT_HIGHEST_BIT) ) + return 1; + + for(uint i=2 ; i::table[i] != mask ) + return 1; + } + + return 0; + } + +#endif + + + +#ifdef TTMATH_PLATFORM64 + + /*! + this method converts the value to a 32 bit unsigned integer + can return a carry if the value is too long to store it in this type + *** this method is created only on a 64 bit platform *** + */ + uint ToUInt(unsigned int & result) const + { + uint c = UInt::ToUInt(result); + + if( c || IsSign() ) + return 1; + + return 0; + } + + + /*! + this method converts the value to a 32 bit unsigned integer + can return a carry if the value is too long to store it in this type + *** this method is created only on a 64 bit platform *** + */ + uint ToInt(unsigned int & result) const + { + return ToUInt(result); + } + + + /*! + this method converts the value to a 32 bit signed integer + can return a carry if the value is too long to store it in this type + *** this method is created only on a 64 bit platform *** + */ + uint ToInt(int & result) const + { + uint first = UInt::table[0]; + + result = int(first); + uint mask = IsSign() ? TTMATH_UINT_MAX_VALUE : 0; + + if( (first >> 31) != (mask >> 31) ) + return 1; + + for(uint i=1 ; i::table[i] != mask ) + return 1; + + return 0; + } + +#endif + + + + /*! + an auxiliary method for converting to a string + */ + template + void ToStringBase(string_type & result, uint b = 10) const + { + if( IsSign() ) + { + Int temp(*this); + temp.Abs(); + temp.UInt::ToStringBase(result, b, true); + } + else + { + UInt::ToStringBase(result, b, false); + } + } + + /*! + this method converts the value to a string with a base equal 'b' + */ + void ToString(std::string & result, uint b = 10) const + { + return ToStringBase(result, b); + } + + + /*! + this method converts the value to a string with a base equal 'b' + */ + std::string ToString(uint b = 10) const + { + std::string result; + ToStringBase(result, b); + + return result; + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + this method converts the value to a string with a base equal 'b' + */ + void ToString(std::wstring & result, uint b = 10) const + { + return ToStringBase(result, b); + } + + + /*! + this method converts the value to a string with a base equal 'b' + */ + std::wstring ToWString(uint b = 10) const + { + std::wstring result; + ToStringBase(result, b); + + return result; + } + +#endif + + + +private: + + /*! + an auxiliary method for converting from a string + */ + template + uint FromStringBase(const char_type * s, uint b = 10, const char_type ** after_source = 0, bool * value_read = 0) + { + bool is_sign = false; + + Misc::SkipWhiteCharacters(s); + + if( *s == '-' ) + { + is_sign = true; + Misc::SkipWhiteCharacters(++s); + } + else + if( *s == '+' ) + { + Misc::SkipWhiteCharacters(++s); + } + + if( UInt::FromString(s,b,after_source,value_read) ) + return 1; + + if( is_sign ) + { + Int mmin; + + mmin.SetMin(); + + /* + the reference to mmin will be automatically converted to the reference + to UInt type + (this value can be equal mmin -- look at a description in ChangeSign()) + */ + if( UInt::operator>( mmin ) ) + return 1; + + /* + if the value is equal mmin the method ChangeSign() does nothing (only returns 1 but we ignore it) + */ + ChangeSign(); + } + else + { + Int mmax; + + mmax.SetMax(); + + if( UInt::operator>( mmax ) ) + return 1; + } + + return 0; + } + + +public: + + /*! + this method converts a string into its value + it returns carry=1 if the value will be too big or an incorrect base 'b' is given + + string is ended with a non-digit value, for example: + "-12" will be translated to -12 + as well as: + "- 12foo" will be translated to -12 too + + existing first white characters will be ommited + (between '-' and a first digit can be white characters too) + + after_source (if exists) is pointing at the end of the parsed string + + value_read (if exists) tells whether something has actually been read (at least one digit) + */ + uint FromString(const char * s, uint b = 10, const char ** after_source = 0, bool * value_read = 0) + { + return FromStringBase(s, b, after_source, value_read); + } + + + /*! + this method converts a string into its value + */ + uint FromString(const wchar_t * s, uint b = 10, const wchar_t ** after_source = 0, bool * value_read = 0) + { + return FromStringBase(s, b, after_source, value_read); + } + + + /*! + this method converts a string into its value + it returns carry=1 if the value will be too big or an incorrect base 'b' is given + */ + uint FromString(const std::string & s, uint b = 10) + { + return FromString( s.c_str(), b ); + } + + + /*! + this operator converts a string into its value (with base = 10) + */ + Int & operator=(const char * s) + { + FromString(s); + + return *this; + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + + /*! + this method converts a string into its value + it returns carry=1 if the value will be too big or an incorrect base 'b' is given + */ + uint FromString(const std::wstring & s, uint b = 10) + { + return FromString( s.c_str(), b ); + } + + + /*! + this operator converts a string into its value (with base = 10) + */ + Int & operator=(const wchar_t * s) + { + FromString(s); + + return *this; + } + + + /*! + this operator converts a string into its value (with base = 10) + */ + Int & operator=(const std::wstring & s) + { + FromString( s.c_str() ); + + return *this; + } + +#endif + + + /*! + this operator converts a string into its value (with base = 10) + */ + Int & operator=(const std::string & s) + { + FromString( s.c_str() ); + + return *this; + } + + + + /*! + * + * methods for comparing + * + * + */ + + bool operator==(const Int & l) const + { + return UInt::operator==(l); + } + + bool operator!=(const Int & l) const + { + return UInt::operator!=(l); + } + + bool operator<(const Int & l) const + { + sint i=value_size-1; + + sint a1 = sint(UInt::table[i]); + sint a2 = sint(l.table[i]); + + if( a1 != a2 ) + return a1 < a2; + + + for(--i ; i>=0 ; --i) + { + if( UInt::table[i] != l.table[i] ) + // comparison as unsigned int + return UInt::table[i] < l.table[i]; + } + + // they're equal + return false; + } + + + bool operator>(const Int & l) const + { + sint i=value_size-1; + + sint a1 = sint(UInt::table[i]); + sint a2 = sint(l.table[i]); + + if( a1 != a2 ) + return a1 > a2; + + + for(--i ; i>=0 ; --i) + { + if( UInt::table[i] != l.table[i] ) + // comparison as unsigned int + return UInt::table[i] > l.table[i]; + } + + // they're equal + return false; + } + + + bool operator<=(const Int & l) const + { + sint i=value_size-1; + + sint a1 = sint(UInt::table[i]); + sint a2 = sint(l.table[i]); + + if( a1 != a2 ) + return a1 < a2; + + + for(--i ; i>=0 ; --i) + { + if( UInt::table[i] != l.table[i] ) + // comparison as unsigned int + return UInt::table[i] < l.table[i]; + } + + // they're equal + return true; + } + + + bool operator>=(const Int & l) const + { + sint i=value_size-1; + + sint a1 = sint(UInt::table[i]); + sint a2 = sint(l.table[i]); + + if( a1 != a2 ) + return a1 > a2; + + + for(--i ; i>=0 ; --i) + { + if( UInt::table[i] != l.table[i] ) + // comparison as unsigned int + return UInt::table[i] > l.table[i]; + } + + // they're equal + return true; + } + + + + /*! + * + * standard mathematical operators + * + */ + + + /*! + an operator for changing the sign + + it's not changing 'this' but the changed value will be returned + */ + Int operator-() const + { + Int temp(*this); + + temp.ChangeSign(); + + return temp; + } + + + Int operator-(const Int & p2) const + { + Int temp(*this); + + temp.Sub(p2); + + return temp; + } + + + Int & operator-=(const Int & p2) + { + Sub(p2); + + return *this; + } + + + Int operator+(const Int & p2) const + { + Int temp(*this); + + temp.Add(p2); + + return temp; + } + + + Int & operator+=(const Int & p2) + { + Add(p2); + + return *this; + } + + + Int operator*(const Int & p2) const + { + Int temp(*this); + + temp.Mul(p2); + + return temp; + } + + + Int & operator*=(const Int & p2) + { + Mul(p2); + + return *this; + } + + + Int operator/(const Int & p2) const + { + Int temp(*this); + + temp.Div(p2); + + return temp; + } + + + Int & operator/=(const Int & p2) + { + Div(p2); + + return *this; + } + + + Int operator%(const Int & p2) const + { + Int temp(*this); + Int remainder; + + temp.Div(p2, remainder); + + return remainder; + } + + + Int & operator%=(const Int & p2) + { + Int remainder; + + Div(p2, remainder); + operator=(remainder); + + return *this; + } + + + /*! + Prefix operator e.g. ++variable + */ + UInt & operator++() + { + AddOne(); + + return *this; + } + + + /*! + Postfix operator e.g. variable++ + */ + UInt operator++(int) + { + UInt temp( *this ); + + AddOne(); + + return temp; + } + + + UInt & operator--() + { + SubOne(); + + return *this; + } + + + UInt operator--(int) + { + UInt temp( *this ); + + SubOne(); + + return temp; + } + + + + /*! + * + * input/output operators for standard streams + * + */ + +private: + + /*! + an auxiliary method for outputing to standard streams + */ + template + static ostream_type & OutputToStream(ostream_type & s, const Int & l) + { + string_type ss; + + l.ToString(ss); + s << ss; + + return s; + } + + + +public: + + + /*! + output to standard streams + */ + friend std::ostream & operator<<(std::ostream & s, const Int & l) + { + return OutputToStream(s, l); + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + output to standard streams + */ + friend std::wostream & operator<<(std::wostream & s, const Int & l) + { + return OutputToStream(s, l); + } + +#endif + + + +private: + + /*! + an auxiliary method for converting from a string + */ + template + static istream_type & InputFromStream(istream_type & s, Int & l) + { + string_type ss; + + // char or wchar_t for operator>> + char_type z; + + // operator>> omits white characters if they're set for ommiting + s >> z; + + if( z=='-' || z=='+' ) + { + ss += z; + s >> z; // we're reading a next character (white characters can be ommited) + } + + // we're reading only digits (base=10) + while( s.good() && Misc::CharToDigit(z, 10)>=0 ) + { + ss += z; + z = static_cast(s.get()); + } + + // we're leaving the last readed character + // (it's not belonging to the value) + s.unget(); + + l.FromString(ss); + + return s; + } + + +public: + + /*! + input from standard streams + */ + friend std::istream & operator>>(std::istream & s, Int & l) + { + return InputFromStream(s, l); + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + input from standard streams + */ + friend std::wistream & operator>>(std::wistream & s, Int & l) + { + return InputFromStream(s, l); + } +#endif + + +}; + +} // namespace + +#endif diff --git a/extern/ttmath/ttmathmisc.h b/extern/ttmath/ttmathmisc.h new file mode 100644 index 0000000000..330a43a468 --- /dev/null +++ b/extern/ttmath/ttmathmisc.h @@ -0,0 +1,250 @@ +/* + * This file is a part of TTMath Bignum Library + * and is distributed under the (new) BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2006-2010, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name Tomasz Sowa nor the names of contributors to this + * project may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef headerfilettmathmisc +#define headerfilettmathmisc + + +/*! + \file ttmathmisc.h + \brief some helpful functions +*/ + + +#include + + +namespace ttmath +{ + +/*! + some helpful functions +*/ +class Misc +{ +public: + + +/* + * + * AssignString(result, str) + * result = str + * + */ + +/*! + result = str +*/ +static void AssignString(std::string & result, const char * str) +{ + result = str; +} + + +#ifndef TTMATH_DONT_USE_WCHAR + +/*! + result = str +*/ +static void AssignString(std::wstring & result, const char * str) +{ + result.clear(); + + for( ; *str ; ++str ) + result += *str; +} + + +/*! + result = str +*/ +static void AssignString(std::wstring & result, const std::string & str) +{ + return AssignString(result, str.c_str()); +} + + +/*! + result = str +*/ +static void AssignString(std::string & result, const wchar_t * str) +{ + result.clear(); + + for( ; *str ; ++str ) + result += static_cast(*str); +} + + +/*! + result = str +*/ +static void AssignString(std::string & result, const std::wstring & str) +{ + return AssignString(result, str.c_str()); +} + +#endif + + +/* + * + * AddString(result, str) + * result += str + * + */ + + +/*! + result += str +*/ +static void AddString(std::string & result, const char * str) +{ + result += str; +} + + +#ifndef TTMATH_DONT_USE_WCHAR + +/*! + result += str +*/ +static void AddString(std::wstring & result, const char * str) +{ + for( ; *str ; ++str ) + result += *str; +} + +#endif + + +/* + this method omits any white characters from the string + char_type is char or wchar_t +*/ +template +static void SkipWhiteCharacters(const char_type * & c) +{ + // 13 is at the end in a DOS text file (\r\n) + while( (*c==' ' ) || (*c=='\t') || (*c==13 ) || (*c=='\n') ) + ++c; +} + + + + +/*! + this static method converts one character into its value + + for example: + 1 -> 1 + 8 -> 8 + A -> 10 + f -> 15 + + this method don't check whether c is correct or not +*/ +static uint CharToDigit(uint c) +{ + if(c>='0' && c<='9') + return c-'0'; + + if(c>='a' && c<='z') + return c-'a'+10; + +return c-'A'+10; +} + + +/*! + this method changes a character 'c' into its value + (if there can't be a correct value it returns -1) + + for example: + c=2, base=10 -> function returns 2 + c=A, base=10 -> function returns -1 + c=A, base=16 -> function returns 10 +*/ +static sint CharToDigit(uint c, uint base) +{ + if( c>='0' && c<='9' ) + c=c-'0'; + else + if( c>='a' && c<='z' ) + c=c-'a'+10; + else + if( c>='A' && c<='Z' ) + c=c-'A'+10; + else + return -1; + + + if( c >= base ) + return -1; + + +return sint(c); +} + + + +/*! + this method converts a digit into a char + digit should be from <0,F> + (we don't have to get a base) + + for example: + 1 -> 1 + 8 -> 8 + 10 -> A + 15 -> F +*/ +static uint DigitToChar(uint digit) +{ + if( digit < 10 ) + return digit + '0'; + +return digit - 10 + 'A'; +} + + +}; // struct Misc + +} + + +#endif diff --git a/extern/ttmath/ttmathobjects.h b/extern/ttmath/ttmathobjects.h new file mode 100644 index 0000000000..c35026bbd0 --- /dev/null +++ b/extern/ttmath/ttmathobjects.h @@ -0,0 +1,809 @@ +/* + * This file is a part of TTMath Mathematical Library + * and is distributed under the (new) BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2006-2010, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name Tomasz Sowa nor the names of contributors to this + * project may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef headerfilettmathobject +#define headerfilettmathobject + +/*! + \file ttmathobjects.h + \brief Mathematic functions. +*/ + +#include +#include +#include +#include + +#include "ttmathtypes.h" +#include "ttmathmisc.h" + + +namespace ttmath +{ + +/*! + objects of this class are used with the mathematical parser + they hold variables or functions defined by a user + + each object has its own table in which we're keeping variables or functions +*/ +class Objects +{ +public: + + + /*! + one item (variable or function) + 'items' will be on the table + */ + struct Item + { + // name of a variable of a function + // internally we store variables and funcions as std::string (not std::wstring even when wide characters are used) + std::string value; + + // number of parameters required by the function + // (if there's a variable this 'param' is ignored) + int param; + + Item() {} + Item(const std::string & v, int p) : value(v), param(p) {} + }; + + // 'Table' is the type of our table + typedef std::map Table; + typedef Table::iterator Iterator; + typedef Table::const_iterator CIterator; + + + + /*! + this method returns true if a character 'c' is a character + which can be in a name + + if 'can_be_digit' is true that means when the 'c' is a digit this + method returns true otherwise it returns false + */ + static bool CorrectCharacter(int c, bool can_be_digit) + { + if( (c>='a' && c<='z') || (c>='A' && c<='Z') ) + return true; + + if( can_be_digit && ((c>='0' && c<='9') || c=='_') ) + return true; + + return false; + } + + + /*! + this method returns true if the name can be as a name of an object + */ + template + static bool IsNameCorrect(const string_type & name) + { + if( name.empty() ) + return false; + + if( !CorrectCharacter(name[0], false) ) + return false; + + typename string_type::const_iterator i = name.begin(); + + for(++i ; i!=name.end() ; ++i) + if( !CorrectCharacter(*i, true) ) + return false; + + return true; + } + + + /*! + this method returns true if such an object is defined (name exists) + */ + bool IsDefined(const std::string & name) + { + Iterator i = table.find(name); + + if( i != table.end() ) + // we have this object in our table + return true; + + return false; + } + + + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + this method returns true if such an object is defined (name exists) + */ + bool IsDefined(const std::wstring & name) + { + // we should check whether the name (in wide characters) are correct + // before calling AssignString() function + if( !IsNameCorrect(name) ) + return false; + + Misc::AssignString(str_tmp1, name); + + return IsDefined(str_tmp1); + } + +#endif + + + /*! + this method adds one object (variable of function) into the table + */ + ErrorCode Add(const std::string & name, const std::string & value, int param = 0) + { + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + Iterator i = table.find(name); + + if( i != table.end() ) + // we have this object in our table + return err_object_exists; + + table.insert( std::make_pair(name, Item(value, param)) ); + + return err_ok; + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + this method adds one object (variable of function) into the table + */ + ErrorCode Add(const std::wstring & name, const std::wstring & value, int param = 0) + { + // we should check whether the name (in wide characters) are correct + // before calling AssignString() function + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + Misc::AssignString(str_tmp1, name); + Misc::AssignString(str_tmp2, value); + + return Add(str_tmp1, str_tmp2, param); + } + +#endif + + + /*! + this method returns 'true' if the table is empty + */ + bool Empty() const + { + return table.empty(); + } + + + /*! + this method clears the table + */ + void Clear() + { + return table.clear(); + } + + + /*! + this method returns 'const_iterator' on the first item on the table + */ + CIterator Begin() const + { + return table.begin(); + } + + + /*! + this method returns 'const_iterator' pointing at the space after last item + (returns table.end()) + */ + CIterator End() const + { + return table.end(); + } + + + /*! + this method changes the value and the number of parameters for a specific object + */ + ErrorCode EditValue(const std::string & name, const std::string & value, int param = 0) + { + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + Iterator i = table.find(name); + + if( i == table.end() ) + return err_unknown_object; + + i->second.value = value; + i->second.param = param; + + return err_ok; + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + + /*! + this method changes the value and the number of parameters for a specific object + */ + ErrorCode EditValue(const std::wstring & name, const std::wstring & value, int param = 0) + { + // we should check whether the name (in wide characters) are correct + // before calling AssignString() function + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + Misc::AssignString(str_tmp1, name); + Misc::AssignString(str_tmp2, value); + + return EditValue(str_tmp1, str_tmp2, param); + } + +#endif + + + /*! + this method changes the name of a specific object + */ + ErrorCode EditName(const std::string & old_name, const std::string & new_name) + { + if( !IsNameCorrect(old_name) || !IsNameCorrect(new_name) ) + return err_incorrect_name; + + Iterator old_i = table.find(old_name); + if( old_i == table.end() ) + return err_unknown_object; + + if( old_name == new_name ) + // the new name is the same as the old one + // we treat it as a normal situation + return err_ok; + + ErrorCode err = Add(new_name, old_i->second.value, old_i->second.param); + + if( err == err_ok ) + { + old_i = table.find(old_name); + TTMATH_ASSERT( old_i != table.end() ) + + table.erase(old_i); + } + + return err; + } + + + +#ifndef TTMATH_DONT_USE_WCHAR + + + /*! + this method changes the name of a specific object + */ + ErrorCode EditName(const std::wstring & old_name, const std::wstring & new_name) + { + // we should check whether the name (in wide characters) are correct + // before calling AssignString() function + if( !IsNameCorrect(old_name) || !IsNameCorrect(new_name) ) + return err_incorrect_name; + + Misc::AssignString(str_tmp1, old_name); + Misc::AssignString(str_tmp2, new_name); + + return EditName(str_tmp1, str_tmp2); + } + +#endif + + + /*! + this method deletes an object + */ + ErrorCode Delete(const std::string & name) + { + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + Iterator i = table.find(name); + + if( i == table.end() ) + return err_unknown_object; + + table.erase( i ); + + return err_ok; + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + + /*! + this method deletes an object + */ + ErrorCode Delete(const std::wstring & name) + { + // we should check whether the name (in wide characters) are correct + // before calling AssignString() function + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + Misc::AssignString(str_tmp1, name); + + return Delete(str_tmp1); + } + +#endif + + + /*! + this method gets the value of a specific object + */ + ErrorCode GetValue(const std::string & name, std::string & value) const + { + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + CIterator i = table.find(name); + + if( i == table.end() ) + { + value.clear(); + return err_unknown_object; + } + + value = i->second.value; + + return err_ok; + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + this method gets the value of a specific object + */ + ErrorCode GetValue(const std::wstring & name, std::wstring & value) + { + // we should check whether the name (in wide characters) are correct + // before calling AssignString() function + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + Misc::AssignString(str_tmp1, name); + ErrorCode err = GetValue(str_tmp1, str_tmp2); + Misc::AssignString(value, str_tmp2); + + return err; + } + +#endif + + + /*! + this method gets the value of a specific object + (this version is used for not copying the whole string) + */ + ErrorCode GetValue(const std::string & name, const char ** value) const + { + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + CIterator i = table.find(name); + + if( i == table.end() ) + { + *value = 0; + return err_unknown_object; + } + + *value = i->second.value.c_str(); + + return err_ok; + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + this method gets the value of a specific object + (this version is used for not copying the whole string) + */ + ErrorCode GetValue(const std::wstring & name, const char ** value) + { + // we should check whether the name (in wide characters) are correct + // before calling AssignString() function + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + Misc::AssignString(str_tmp1, name); + + return GetValue(str_tmp1, value); + } + +#endif + + + /*! + this method gets the value and the number of parameters + of a specific object + */ + ErrorCode GetValueAndParam(const std::string & name, std::string & value, int * param) const + { + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + CIterator i = table.find(name); + + if( i == table.end() ) + { + value.empty(); + *param = 0; + return err_unknown_object; + } + + value = i->second.value; + *param = i->second.param; + + return err_ok; + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + this method gets the value and the number of parameters + of a specific object + */ + ErrorCode GetValueAndParam(const std::wstring & name, std::wstring & value, int * param) + { + // we should check whether the name (in wide characters) are correct + // before calling AssignString() function + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + Misc::AssignString(str_tmp1, name); + ErrorCode err = GetValueAndParam(str_tmp1, str_tmp2, param); + Misc::AssignString(value, str_tmp2); + + return err; + } + +#endif + + + /*! + this method sets the value and the number of parameters + of a specific object + (this version is used for not copying the whole string) + */ + ErrorCode GetValueAndParam(const std::string & name, const char ** value, int * param) const + { + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + CIterator i = table.find(name); + + if( i == table.end() ) + { + *value = 0; + *param = 0; + return err_unknown_object; + } + + *value = i->second.value.c_str(); + *param = i->second.param; + + return err_ok; + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + + /*! + this method sets the value and the number of parameters + of a specific object + (this version is used for not copying the whole string + but in fact we make one copying during AssignString()) + */ + ErrorCode GetValueAndParam(const std::wstring & name, const char ** value, int * param) + { + // we should check whether the name (in wide characters) are correct + // before calling AssignString() function + if( !IsNameCorrect(name) ) + return err_incorrect_name; + + Misc::AssignString(str_tmp1, name); + + return GetValueAndParam(str_tmp1, value, param); + } + + +#endif + + + /*! + this method returns a pointer into the table + */ + Table * GetTable() + { + return &table; + } + + +private: + + Table table; + std::string str_tmp1, str_tmp2; + +}; // end of class Objects + + + + + + + +/*! + objects of the class History are used to keep values in functions + which take a lot of time during calculating, for instance in the + function Factorial(x) + + it means that when we're calculating e.g. Factorial(1000) and the + Factorial finds that we have calculated it before, the value (result) + is taken from the history +*/ +template +class History +{ + /*! + one item in the History's object holds a key, a value for the key + and a corresponding error code + */ + struct Item + { + ValueType key, value; + ErrorCode err; + }; + + + /*! + we use std::list for simply deleting the first item + but because we're searching through the whole container + (in the method Get) the container should not be too big + (linear time of searching) + */ + typedef std::list buffer_type; + buffer_type buffer; + typename buffer_type::size_type buffer_max_size; + +public: + + /*! + default constructor + default max size of the History's container is 15 items + */ + History() + { + buffer_max_size = 15; + } + + + /*! + a constructor which takes another value of the max size + of the History's container + */ + History(typename buffer_type::size_type new_size) + { + buffer_max_size = new_size; + } + + + /*! + this method adds one item into the History + if the size of the container is greater than buffer_max_size + the first item will be removed + */ + void Add(const ValueType & key, const ValueType & value, ErrorCode err) + { + Item item; + item.key = key; + item.value = value; + item.err = err; + + buffer.insert( buffer.end(), item ); + + if( buffer.size() > buffer_max_size ) + buffer.erase(buffer.begin()); + } + + + /*! + this method checks whether we have an item which has the key equal 'key' + + if there's such item the method sets the 'value' and the 'err' + and returns true otherwise it returns false and 'value' and 'err' + remain unchanged + */ + bool Get(const ValueType & key, ValueType & value, ErrorCode & err) + { + typename buffer_type::iterator i = buffer.begin(); + + for( ; i != buffer.end() ; ++i ) + { + if( i->key == key ) + { + value = i->value; + err = i->err; + return true; + } + } + + return false; + } + + + /*! + this methods deletes an item + + we assume that there is only one item with the 'key' + (this methods removes the first one) + */ + bool Remove(const ValueType & key) + { + typename buffer_type::iterator i = buffer.begin(); + + for( ; i != buffer.end() ; ++i ) + { + if( i->key == key ) + { + buffer.erase(i); + return true; + } + } + + return false; + } + + +}; // end of class History + + + +/*! + this is an auxiliary class used when calculating Gamma() or Factorial() + + in multithreaded environment you can provide an object of this class to + the Gamma() or Factorial() function, e.g; + typedef Big<1, 3> MyBig; + MyBig x = 123456; + CGamma cgamma; + std::cout << Gamma(x, cgamma); + each thread should have its own CGamma<> object + + in a single-thread environment a CGamma<> object is a static variable + in a second version of Gamma() and you don't have to explicitly use it, e.g. + typedef Big<1, 3> MyBig; + MyBig x = 123456; + std::cout << Gamma(x); +*/ +template +struct CGamma +{ + /*! + this table holds factorials + 1 + 1 + 2 + 6 + 24 + 120 + 720 + ....... + */ + std::vector fact; + + + /*! + this table holds Bernoulli numbers + 1 + -0.5 + 0.166666666666666666666666667 + 0 + -0.0333333333333333333333333333 + 0 + 0.0238095238095238095238095238 + 0 + -0.0333333333333333333333333333 + 0 + 0.075757575757575757575757576 + ..... + */ + std::vector bern; + + + /*! + here we store some calculated values + (this is for speeding up, if the next argument of Gamma() or Factorial() + is in the 'history' then the result we are not calculating but simply + return from the 'history' object) + */ + History history; + + + /*! + this method prepares some coefficients: factorials and Bernoulli numbers + stored in 'fact' and 'bern' objects + + how many values should be depends on the size of the mantissa - if + the mantissa is larger then we must calculate more values + for a mantissa which consists of 256 bits (8 words on a 32bit platform) + we have to calculate about 30 values (the size of fact and bern will be 30), + and for a 2048 bits mantissa we have to calculate 306 coefficients + + you don't have to call this method, these coefficients will be automatically calculated + when they are needed + + you must note that calculating these coefficients is a little time-consuming operation, + (especially when the mantissa is large) and first call to Gamma() or Factorial() + can take more time than next calls, and in the end this is the point when InitAll() + comes in handy: you can call this method somewhere at the beginning of your program + */ + void InitAll(); + // definition is in ttmath.h +}; + + + + +} // namespace + +#endif diff --git a/extern/ttmath/ttmaththreads.h b/extern/ttmath/ttmaththreads.h new file mode 100644 index 0000000000..586227f2fc --- /dev/null +++ b/extern/ttmath/ttmaththreads.h @@ -0,0 +1,250 @@ +/* + * This file is a part of TTMath Bignum Library + * and is distributed under the (new) BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2006-2009, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name Tomasz Sowa nor the names of contributors to this + * project may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + + + +#ifndef headerfilettmaththreads +#define headerfilettmaththreads + +#include "ttmathtypes.h" + +#ifdef TTMATH_WIN32_THREADS +#include +#include +#endif + +#ifdef TTMATH_POSIX_THREADS +#include +#endif + + + +/*! + \file ttmaththreads.h + \brief Some objects used in multithreads environment +*/ + + +/* + this is a simple skeleton of a program in multithreads environment: + + #define TTMATH_MULTITHREADS + #include + + TTMATH_MULTITHREADS_HELPER + + int main() + { + [...] + } + + make sure that macro TTMATH_MULTITHREADS is defined and (somewhere in *.cpp file) + use TTMATH_MULTITHREADS_HELPER macro (outside of any classes/functions/namespaces scope) +*/ + + +namespace ttmath +{ + + +#ifdef TTMATH_WIN32_THREADS + + /* + we use win32 threads + */ + + + /*! + in multithreads environment you should use TTMATH_MULTITHREADS_HELPER macro + somewhere in *.cpp file + + (at the moment in win32 this macro does nothing) + */ + #define TTMATH_MULTITHREADS_HELPER + + + /*! + objects of this class are used to synchronize + */ + class ThreadLock + { + HANDLE mutex_handle; + + + void CreateName(char * buffer) const + { + #ifdef _MSC_VER + #pragma warning (disable : 4996) + // warning C4996: 'sprintf': This function or variable may be unsafe. Consider using sprintf_s instead. + #endif + + sprintf(buffer, "TTMATH_LOCK_%ul", (unsigned long)GetCurrentProcessId()); + + #ifdef _MSC_VER + #pragma warning (default : 4996) + #endif + } + + + public: + + bool Lock() + { + char buffer[50]; + + CreateName(buffer); + mutex_handle = CreateMutexA(0, false, buffer); + + if( mutex_handle == 0 ) + return false; + + WaitForSingleObject(mutex_handle, INFINITE); + + return true; + } + + + ThreadLock() + { + mutex_handle = 0; + } + + + ~ThreadLock() + { + if( mutex_handle != 0 ) + { + ReleaseMutex(mutex_handle); + CloseHandle(mutex_handle); + } + } + }; + +#endif // #ifdef TTMATH_WIN32_THREADS + + + + + +#ifdef TTMATH_POSIX_THREADS + + /* + we use posix threads + */ + + + /*! + in multithreads environment you should use TTMATH_MULTITHREADS_HELPER macro + somewhere in *.cpp file + (this macro defines a pthread_mutex_t object used by TTMath library) + */ + #define TTMATH_MULTITHREADS_HELPER \ + namespace ttmath \ + { \ + pthread_mutex_t ttmath_mutex = PTHREAD_MUTEX_INITIALIZER; \ + } + + + /*! + ttmath_mutex will be defined by TTMATH_MULTITHREADS_HELPER macro + */ + extern pthread_mutex_t ttmath_mutex; + + + /*! + objects of this class are used to synchronize + */ + class ThreadLock + { + public: + + bool Lock() + { + if( pthread_mutex_lock(&ttmath_mutex) != 0 ) + return false; + + return true; + } + + + ~ThreadLock() + { + pthread_mutex_unlock(&ttmath_mutex); + } + }; + +#endif // #ifdef TTMATH_POSIX_THREADS + + + + +#if !defined(TTMATH_POSIX_THREADS) && !defined(TTMATH_WIN32_THREADS) + + /*! + we don't use win32 and pthreads + */ + + /*! + */ + #define TTMATH_MULTITHREADS_HELPER + + + /*! + objects of this class are used to synchronize + actually we don't synchronize, the method Lock() returns always 'false' + */ + class ThreadLock + { + public: + + bool Lock() + { + return false; + } + }; + + +#endif // #if !defined(TTMATH_POSIX_THREADS) && !defined(TTMATH_WIN32_THREADS) + + + + + +} // namespace + +#endif + diff --git a/extern/ttmath/ttmathtypes.h b/extern/ttmath/ttmathtypes.h new file mode 100644 index 0000000000..3d9ddbe7b0 --- /dev/null +++ b/extern/ttmath/ttmathtypes.h @@ -0,0 +1,676 @@ +/* + * This file is a part of TTMath Bignum Library + * and is distributed under the (new) BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2006-2012, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name Tomasz Sowa nor the names of contributors to this + * project may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef headerfilettmathtypes +#define headerfilettmathtypes + +/*! + \file ttmathtypes.h + \brief constants used in the library + + As our library is written in header files (templates) we cannot use + constants like 'const int' etc. because we should have some source files + *.cpp to define this variables. Only what we can have are constants + defined by #define preprocessor macros. + + All macros are preceded by TTMATH_ prefix +*/ + + +#include +#include +#include + +#ifndef _MSC_VER +#include +// for uint64_t and int64_t on a 32 bit platform +#endif + + + +/*! + the version of the library + + TTMATH_PRERELEASE_VER is either zero or one + zero means that this is the release version of the library + (one means something like beta) +*/ +#define TTMATH_MAJOR_VER 0 +#define TTMATH_MINOR_VER 9 +#define TTMATH_REVISION_VER 3 + +#define TTMATH_PRERELEASE_VER 0 + + + +/*! + you can define a platform explicitly by defining either + TTMATH_PLATFORM32 or TTMATH_PLATFORM64 macro +*/ +#if !defined TTMATH_PLATFORM32 && !defined TTMATH_PLATFORM64 + + #if !defined _M_X64 && !defined __x86_64__ + + /* + other platforms than x86 and amd64 are not recognized at the moment + so you should set TTMATH_PLATFORMxx manually + */ + + // we're using a 32bit platform + #define TTMATH_PLATFORM32 + + #else + + // we're using a 64bit platform + #define TTMATH_PLATFORM64 + + #endif + +#endif + + +/*! + asm version of the library is available by default only for: + x86 and amd64 platforms and for Microsoft Visual and GCC compilers + + but you can force using asm version (the same asm as for Microsoft Visual) + by defining TTMATH_FORCEASM macro + you have to be sure that your compiler accept such an asm format +*/ +#ifndef TTMATH_FORCEASM + + #if !defined __i386__ && !defined _X86_ && !defined _M_IX86 && !defined __x86_64__ && !defined _M_X64 + /*! + x86 architecture: + __i386__ defined by GNU C + _X86_ defined by MinGW32 + _M_IX86 defined by Visual Studio, Intel C/C++, Digital Mars and Watcom C/C++ + + amd64 architecture: + __x86_64__ defined by GNU C, CLANG (LLVM) and Sun Studio + _M_X64 defined by Visual Studio + + asm version is available only for x86 or amd64 platforms + */ + #define TTMATH_NOASM + #endif + + + + #if !defined _MSC_VER && !defined __GNUC__ + /*! + another compilers than MS VC or GCC or CLANG (LLVM) by default use no asm version + (CLANG defines __GNUC__ too) + */ + #define TTMATH_NOASM + #endif + +#endif + + +namespace ttmath +{ + + +#ifdef TTMATH_PLATFORM32 + + /*! + on 32bit platforms one word (uint, sint) will be equal 32bits + */ + typedef unsigned int uint; + typedef signed int sint; + + /*! + on 32 bit platform ulint and slint will be equal 64 bits + */ + #ifdef _MSC_VER + // long long on MS Windows (Visual and GCC mingw compilers) have 64 bits + // stdint.h is not available on Visual Studio prior to VS 2010 version + typedef unsigned long long int ulint; + typedef signed long long int slint; + #else + // we do not use 'long' here because there is a difference in unix and windows + // environments: in unix 'long' has 64 bits but in windows it has only 32 bits + typedef uint64_t ulint; + typedef int64_t slint; + #endif + + /*! + how many bits there are in the uint type + */ + #define TTMATH_BITS_PER_UINT 32u + + /*! + the mask for the highest bit in the unsigned 32bit word (2^31) + */ + #define TTMATH_UINT_HIGHEST_BIT 2147483648u + + /*! + the max value of the unsigned 32bit word (2^32 - 1) + (all bits equal one) + */ + #define TTMATH_UINT_MAX_VALUE 4294967295u + + /*! + the number of words (32bit words on 32bit platform) + which are kept in built-in variables for a Big<> type + (these variables are defined in ttmathbig.h) + */ + #define TTMATH_BUILTIN_VARIABLES_SIZE 256u + + /*! + this macro returns the number of machine words + capable to hold min_bits bits + e.g. TTMATH_BITS(128) returns 4 + */ + #define TTMATH_BITS(min_bits) ((min_bits-1)/32 + 1) + +#else + + /*! + on 64bit platforms one word (uint, sint) will be equal 64bits + */ + #ifdef _MSC_VER + /* in VC 'long' type has 32 bits, __int64 is VC extension */ + typedef unsigned __int64 uint; + typedef signed __int64 sint; + #else + typedef unsigned long uint; + typedef signed long sint; + #endif + + /*! + on 64bit platforms we do not define ulint and slint + */ + + /*! + how many bits there are in the uint type + */ + #define TTMATH_BITS_PER_UINT 64ul + + /*! + the mask for the highest bit in the unsigned 64bit word (2^63) + */ + #define TTMATH_UINT_HIGHEST_BIT 9223372036854775808ul + + /*! + the max value of the unsigned 64bit word (2^64 - 1) + (all bits equal one) + */ + #define TTMATH_UINT_MAX_VALUE 18446744073709551615ul + + /*! + the number of words (64bit words on 64bit platforms) + which are kept in built-in variables for a Big<> type + (these variables are defined in ttmathbig.h) + */ + #define TTMATH_BUILTIN_VARIABLES_SIZE 128ul + + /*! + this macro returns the number of machine words + capable to hold min_bits bits + e.g. TTMATH_BITS(128) returns 2 + */ + #define TTMATH_BITS(min_bits) ((min_bits-1)/64 + 1) + +#endif +} + + +#if defined(TTMATH_MULTITHREADS) && !defined(TTMATH_MULTITHREADS_NOSYNC) + #if !defined(TTMATH_POSIX_THREADS) && !defined(TTMATH_WIN32_THREADS) + + #if defined(_WIN32) + #define TTMATH_WIN32_THREADS + #elif defined(unix) || defined(__unix__) || defined(__unix) + #define TTMATH_POSIX_THREADS + #endif + + #endif +#endif + + + +/*! + this variable defines how many iterations are performed + during some kind of calculating when we're making any long formulas + (for example Taylor series) + + it's used in ExpSurrounding0(...), LnSurrounding1(...), Sin0pi05(...), etc. + + note! there'll not be so many iterations, iterations are stopped when + there is no sense to continue calculating (for example when the result + still remains unchanged after adding next series and we know that the next + series are smaller than previous ones) +*/ +#define TTMATH_ARITHMETIC_MAX_LOOP 10000 + + + +/*! + this is a limit when calculating Karatsuba multiplication + if the size of a vector is smaller than TTMATH_USE_KARATSUBA_MULTIPLICATION_FROM_SIZE + the Karatsuba algorithm will use standard schoolbook multiplication +*/ +#ifdef TTMATH_DEBUG_LOG + // if TTMATH_DEBUG_LOG is defined then we should use the same size regardless of the compiler + #define TTMATH_USE_KARATSUBA_MULTIPLICATION_FROM_SIZE 3 +#else + #ifdef __GNUC__ + #define TTMATH_USE_KARATSUBA_MULTIPLICATION_FROM_SIZE 3 + #else + #define TTMATH_USE_KARATSUBA_MULTIPLICATION_FROM_SIZE 5 + #endif +#endif + + +/*! + this is a special value used when calculating the Gamma(x) function + if x is greater than this value then the Gamma(x) will be calculated using + some kind of series + + don't use smaller values than about 100 +*/ +#define TTMATH_GAMMA_BOUNDARY 2000 + + + + + +namespace ttmath +{ + + /*! + lib type codes: + asm_vc_32 - with asm code designed for Microsoft Visual C++ (32 bits) + asm_gcc_32 - with asm code designed for GCC (32 bits) + asm_vc_64 - with asm for VC (64 bit) + asm_gcc_64 - with asm for GCC (64 bit) + no_asm_32 - pure C++ version (32 bit) - without any asm code + no_asm_64 - pure C++ version (64 bit) - without any asm code + */ + enum LibTypeCode + { + asm_vc_32 = 0, + asm_gcc_32, + asm_vc_64, + asm_gcc_64, + no_asm_32, + no_asm_64 + }; + + + /*! + error codes + */ + enum ErrorCode + { + err_ok = 0, + err_nothing_has_read, + err_unknown_character, + err_unexpected_final_bracket, + err_stack_not_clear, + err_unknown_variable, + err_division_by_zero, + err_interrupt, + err_overflow, + err_unknown_function, + err_unknown_operator, + err_unexpected_semicolon_operator, + err_improper_amount_of_arguments, + err_improper_argument, + err_unexpected_end, + err_internal_error, + err_incorrect_name, + err_incorrect_value, + err_variable_exists, + err_variable_loop, + err_functions_loop, + err_must_be_only_one_value, + err_object_exists, + err_unknown_object, + err_still_calculating, + err_in_short_form_used_function, + err_percent_from + }; + + + /*! + this struct is used when converting to/from a string + /temporarily only in Big::ToString() and Big::FromString()/ + */ + struct Conv + { + /*! + base (radix) on which the value will be shown (or read) + default: 10 + */ + uint base; + + + /*! + used only in Big::ToString() + if true the value will be always shown in the scientific mode, e.g: 123e+30 + default: false + */ + bool scient; + + + /*! + used only in Big::ToString() + if scient is false then the value will be printed in the scientific mode + only if the exponent is greater than scien_from + default: 15 + */ + sint scient_from; + + + /*! + if 'base_round' is true and 'base' is different from 2, 4, 8, or 16 + and the result value is not an integer then we make an additional rounding + (after converting the last digit from the result is skipped) + default: true + + e.g. + Conv c; + c.base_round = false; + Big<1, 1> a = "0.1"; // decimal input + std::cout << a.ToString(c) << std::endl; // the result is: 0.099999999 + */ + bool base_round; + + + /*! + used only in Big::ToString() + tells how many digits after comma are possible + default: -1 which means all digits are printed + + set it to zero if you want integer value only + + for example when the value is: + 12.345678 and 'round' is 4 + then the result will be + 12.3457 (the last digit was rounded) + */ + sint round; + + + /*! + if true that not mattered digits in the mantissa will be cut off + (zero characters at the end -- after the comma operator) + e.g. 1234,78000 will be: 1234,78 + default: true + */ + bool trim_zeroes; + + + /*! + the main comma operator (used when reading and writing) + default is a dot '.' + */ + uint comma; + + + /*! + additional comma operator (used only when reading) + if you don't want it just set it to zero + default is a comma ',' + + this allowes you to convert from a value: + 123.45 as well as from 123,45 + */ + uint comma2; + + + /*! + it sets the character which is used for grouping + if group=' ' then: 1234,56789 will be printed as: 1 234,567 89 + + if you don't want grouping just set it to zero (which is default) + */ + uint group; + + + /*! + how many digits should be grouped (it is used if 'group' is non zero) + default: 3 + */ + uint group_digits; + + + /*! + */ + uint group_exp; // not implemented yet + + + + + Conv() + { + // default values + base = 10; + scient = false; + scient_from = 15; + base_round = true; + round = -1; + trim_zeroes = true; + comma = '.'; + comma2 = ','; + group = 0; + group_digits = 3; + group_exp = 0; + } + }; + + + + /*! + this simple class can be used in multithreading model + (you can write your own class derived from this one) + + for example: in some functions like Factorial() + /at the moment only Factorial/ you can give a pointer to + the 'stop object', if the method WasStopSignal() of this + object returns true that means we should break the calculating + and return + */ + class StopCalculating + { + public: + virtual bool WasStopSignal() const volatile { return false; } + virtual ~StopCalculating(){} + }; + + + /*! + a small class which is useful when compiling with gcc + + object of this type holds the name and the line of a file + in which the macro TTMATH_ASSERT or TTMATH_REFERENCE_ASSERT was used + */ + class ExceptionInfo + { + const char * file; + int line; + + public: + ExceptionInfo() : file(0), line(0) {} + ExceptionInfo(const char * f, int l) : file(f), line(l) {} + + std::string Where() const + { + if( !file ) + return "unknown"; + + std::ostringstream result; + result << file << ":" << line; + + return result.str(); + } + }; + + + /*! + A small class used for reporting 'reference' errors + + In the library is used macro TTMATH_REFERENCE_ASSERT which + can throw an exception of this type + + ** from version 0.9.2 this macro is removed from all methods + in public interface so you don't have to worry about it ** + + If you compile with gcc you can get a small benefit + from using method Where() (it returns std::string) with + the name and the line of a file where the macro TTMATH_REFERENCE_ASSERT + was used) + */ + class ReferenceError : public std::logic_error, public ExceptionInfo + { + public: + + ReferenceError() : std::logic_error("reference error") + { + } + + ReferenceError(const char * f, int l) : + std::logic_error("reference error"), ExceptionInfo(f,l) + { + } + + std::string Where() const + { + return ExceptionInfo::Where(); + } + }; + + + /*! + a small class used for reporting errors + + in the library is used macro TTMATH_ASSERT which + (if the condition in it is false) throw an exception + of this type + + if you compile with gcc you can get a small benefit + from using method Where() (it returns std::string) with + the name and the line of a file where the macro TTMATH_ASSERT + was used) + */ + class RuntimeError : public std::runtime_error, public ExceptionInfo + { + public: + + RuntimeError() : std::runtime_error("internal error") + { + } + + RuntimeError(const char * f, int l) : + std::runtime_error("internal error"), ExceptionInfo(f,l) + { + } + + std::string Where() const + { + return ExceptionInfo::Where(); + } + }; + + + + /*! + TTMATH_DEBUG + this macro enables further testing during writing your code + you don't have to define it in a release mode + + if this macro is set then macros TTMATH_ASSERT and TTMATH_REFERENCE_ASSERT + are set as well and these macros can throw an exception if a condition in it + is not fulfilled (look at the definition of TTMATH_ASSERT and TTMATH_REFERENCE_ASSERT) + + TTMATH_DEBUG is set automatically if DEBUG or _DEBUG are defined + */ + #if defined DEBUG || defined _DEBUG + #define TTMATH_DEBUG + #endif + + + #ifdef TTMATH_DEBUG + + #if defined(__FILE__) && defined(__LINE__) + + #define TTMATH_REFERENCE_ASSERT(expression) \ + if( &(expression) == this ) throw ttmath::ReferenceError(__FILE__, __LINE__); + + #define TTMATH_ASSERT(expression) \ + if( !(expression) ) throw ttmath::RuntimeError(__FILE__, __LINE__); + + #else + + #define TTMATH_REFERENCE_ASSERT(expression) \ + if( &(expression) == this ) throw ReferenceError(); + + #define TTMATH_ASSERT(expression) \ + if( !(expression) ) throw RuntimeError(); + #endif + + #else + #define TTMATH_REFERENCE_ASSERT(expression) + #define TTMATH_ASSERT(expression) + #endif + + + + #ifdef TTMATH_DEBUG_LOG + #define TTMATH_LOG(msg) PrintLog(msg, std::cout); + #define TTMATH_LOGC(msg, carry) PrintLog(msg, carry, std::cout); + #define TTMATH_VECTOR_LOG(msg, vector, len) PrintVectorLog(msg, std::cout, vector, len); + #define TTMATH_VECTOR_LOGC(msg, carry, vector, len) PrintVectorLog(msg, carry, std::cout, vector, len); + #else + #define TTMATH_LOG(msg) + #define TTMATH_LOGC(msg, carry) + #define TTMATH_VECTOR_LOG(msg, vector, len) + #define TTMATH_VECTOR_LOGC(msg, carry, vector, len) + #endif + + + + +} // namespace + + +#endif + diff --git a/extern/ttmath/ttmathuint.h b/extern/ttmath/ttmathuint.h new file mode 100644 index 0000000000..b9cf67cd11 --- /dev/null +++ b/extern/ttmath/ttmathuint.h @@ -0,0 +1,4126 @@ +/* + * This file is a part of TTMath Bignum Library + * and is distributed under the (new) BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2006-2011, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name Tomasz Sowa nor the names of contributors to this + * project may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + + + +#ifndef headerfilettmathuint +#define headerfilettmathuint + + +/*! + \file ttmathuint.h + \brief template class UInt +*/ + +#include +#include + + +#include "ttmathtypes.h" +#include "ttmathmisc.h" + + + +/*! + \brief a namespace for the TTMath library +*/ +namespace ttmath +{ + +/*! + \brief UInt implements a big integer value without a sign + + value_size - how many bytes specify our value + on 32bit platforms: value_size=1 -> 4 bytes -> 32 bits + on 64bit platforms: value_size=1 -> 8 bytes -> 64 bits + value_size = 1,2,3,4,5,6.... +*/ +template +class UInt +{ +public: + + /*! + buffer for the integer value + table[0] - the lowest word of the value + */ + uint table[value_size]; + + + + /*! + some methods used for debugging purposes + */ + + + /*! + this method is used when macro TTMATH_DEBUG_LOG is defined + */ + template + static void PrintVectorLog(const char_type * msg, ostream_type & output, const uint * vector, uint vector_len) + { + output << msg << std::endl; + + for(uint i=0 ; i + static void PrintVectorLog(const char_type * msg, uint carry, ostream_type & output, const uint * vector, uint vector_len) + { + PrintVectorLog(msg, output, vector, vector_len); + output << " carry: " << carry << std::endl; + } + + + /*! + this method is used when macro TTMATH_DEBUG_LOG is defined + */ + template + void PrintLog(const char_type * msg, ostream_type & output) const + { + PrintVectorLog(msg, output, table, value_size); + } + + + /*! + this method is used when macro TTMATH_DEBUG_LOG is defined + */ + template + void PrintLog(const char_type * msg, uint carry, ostream_type & output) const + { + PrintVectorLog(msg, output, table, value_size); + output << " carry: " << carry << std::endl; + } + + + /*! + this method returns the size of the table + */ + uint Size() const + { + return value_size; + } + + + /*! + this method sets zero + */ + void SetZero() + { + // in the future here can be 'memset' + + for(uint i=0 ; i & ss2) + { + for(uint i=0 ; i=0 && temp_table_index=0 ; --i) + table[i] = 0; + + + TTMATH_LOG("UInt::SetFromTable") + } + +#endif + + +#ifdef TTMATH_PLATFORM64 + /*! + this method copies the value stored in an another table + (warning: first values in temp_table are the highest words -- it's different + from our table) + + ***this method is created only on a 64bit platform*** + + we copy as many words as it is possible + + if temp_table_len is bigger than value_size we'll try to round + the lowest word from table depending on the last not used bit in temp_table + (this rounding isn't a perfect rounding -- look at the description below) + + and if temp_table_len is smaller than value_size we'll clear the rest words + in the table + + warning: we're using 'temp_table' as a pointer at 32bit words + */ + void SetFromTable(const unsigned int * temp_table, uint temp_table_len) + { + uint temp_table_index = 0; + sint i; // 'i' with a sign + + for(i=value_size-1 ; i>=0 && temp_table_index= 0 ; --i) + table[i] = 0; + + TTMATH_LOG("UInt::SetFromTable") + } + +#endif + + + + + + /*! + * + * basic mathematic functions + * + */ + + + + + /*! + this method adds one to the existing value + */ + uint AddOne() + { + return AddInt(1); + } + + + /*! + this method subtracts one from the existing value + */ + uint SubOne() + { + return SubInt(1); + } + + +private: + + + /*! + an auxiliary method for moving bits into the left hand side + + this method moves only words + */ + void RclMoveAllWords(uint & rest_bits, uint & last_c, uint bits, uint c) + { + rest_bits = bits % TTMATH_BITS_PER_UINT; + uint all_words = bits / TTMATH_BITS_PER_UINT; + uint mask = ( c ) ? TTMATH_UINT_MAX_VALUE : 0; + + + if( all_words >= value_size ) + { + if( all_words == value_size && rest_bits == 0 ) + last_c = table[0] & 1; + // else: last_c is default set to 0 + + // clearing + for(uint i = 0 ; i 0 ) + { + // 0 < all_words < value_size + + sint first, second; + last_c = table[value_size - all_words] & 1; // all_words is greater than 0 + + // copying the first part of the value + for(first = value_size-1, second=first-all_words ; second>=0 ; --first, --second) + table[first] = table[second]; + + // setting the rest to 'c' + for( ; first>=0 ; --first ) + table[first] = mask; + } + + TTMATH_LOG("UInt::RclMoveAllWords") + } + +public: + + /*! + moving all bits into the left side 'bits' times + return value <- this <- C + + bits is from a range of <0, man * TTMATH_BITS_PER_UINT> + or it can be even bigger then all bits will be set to 'c' + + the value c will be set into the lowest bits + and the method returns state of the last moved bit + */ + uint Rcl(uint bits, uint c=0) + { + uint last_c = 0; + uint rest_bits = bits; + + if( bits == 0 ) + return 0; + + if( bits >= TTMATH_BITS_PER_UINT ) + RclMoveAllWords(rest_bits, last_c, bits, c); + + if( rest_bits == 0 ) + { + TTMATH_LOG("UInt::Rcl") + return last_c; + } + + // rest_bits is from 1 to TTMATH_BITS_PER_UINT-1 now + if( rest_bits == 1 ) + { + last_c = Rcl2_one(c); + } + else if( rest_bits == 2 ) + { + // performance tests showed that for rest_bits==2 it's better to use Rcl2_one twice instead of Rcl2(2,c) + Rcl2_one(c); + last_c = Rcl2_one(c); + } + else + { + last_c = Rcl2(rest_bits, c); + } + + TTMATH_LOGC("UInt::Rcl", last_c) + + return last_c; + } + +private: + + /*! + an auxiliary method for moving bits into the right hand side + + this method moves only words + */ + void RcrMoveAllWords(uint & rest_bits, uint & last_c, uint bits, uint c) + { + rest_bits = bits % TTMATH_BITS_PER_UINT; + uint all_words = bits / TTMATH_BITS_PER_UINT; + uint mask = ( c ) ? TTMATH_UINT_MAX_VALUE : 0; + + + if( all_words >= value_size ) + { + if( all_words == value_size && rest_bits == 0 ) + last_c = (table[value_size-1] & TTMATH_UINT_HIGHEST_BIT) ? 1 : 0; + // else: last_c is default set to 0 + + // clearing + for(uint i = 0 ; i 0 ) + { + // 0 < all_words < value_size + + uint first, second; + last_c = (table[all_words - 1] & TTMATH_UINT_HIGHEST_BIT) ? 1 : 0; // all_words is > 0 + + // copying the first part of the value + for(first=0, second=all_words ; second this -> return value + + bits is from a range of <0, man * TTMATH_BITS_PER_UINT> + or it can be even bigger then all bits will be set to 'c' + + the value c will be set into the highest bits + and the method returns state of the last moved bit + */ + uint Rcr(uint bits, uint c=0) + { + uint last_c = 0; + uint rest_bits = bits; + + if( bits == 0 ) + return 0; + + if( bits >= TTMATH_BITS_PER_UINT ) + RcrMoveAllWords(rest_bits, last_c, bits, c); + + if( rest_bits == 0 ) + { + TTMATH_LOG("UInt::Rcr") + return last_c; + } + + // rest_bits is from 1 to TTMATH_BITS_PER_UINT-1 now + if( rest_bits == 1 ) + { + last_c = Rcr2_one(c); + } + else if( rest_bits == 2 ) + { + // performance tests showed that for rest_bits==2 it's better to use Rcr2_one twice instead of Rcr2(2,c) + Rcr2_one(c); + last_c = Rcr2_one(c); + } + else + { + last_c = Rcr2(rest_bits, c); + } + + TTMATH_LOGC("UInt::Rcr", last_c) + + return last_c; + } + + + /*! + this method moves all bits into the left side + (it returns value how many bits have been moved) + */ + uint CompensationToLeft() + { + uint moving = 0; + + // a - index a last word which is different from zero + sint a; + for(a=value_size-1 ; a>=0 && table[a]==0 ; --a); + + if( a < 0 ) + return moving; // all words in table have zero + + if( a != value_size-1 ) + { + moving += ( value_size-1 - a ) * TTMATH_BITS_PER_UINT; + + // moving all words + sint i; + for(i=value_size-1 ; a>=0 ; --i, --a) + table[i] = table[a]; + + // setting the rest word to zero + for(; i>=0 ; --i) + table[i] = 0; + } + + uint moving2 = FindLeadingBitInWord( table[value_size-1] ); + // moving2 is different from -1 because the value table[value_size-1] + // is not zero + + moving2 = TTMATH_BITS_PER_UINT - moving2 - 1; + Rcl(moving2); + + TTMATH_LOG("UInt::CompensationToLeft") + + return moving + moving2; + } + + + /*! + this method looks for the highest set bit + + result: + if 'this' is not zero: + return value - true + 'table_id' - the index of a word <0..value_size-1> + 'index' - the index of this set bit in the word <0..TTMATH_BITS_PER_UINT) + + if 'this' is zero: + return value - false + both 'table_id' and 'index' are zero + */ + bool FindLeadingBit(uint & table_id, uint & index) const + { + for(table_id=value_size-1 ; table_id!=0 && table[table_id]==0 ; --table_id); + + if( table_id==0 && table[table_id]==0 ) + { + // is zero + index = 0; + + return false; + } + + // table[table_id] is different from 0 + index = FindLeadingBitInWord( table[table_id] ); + + return true; + } + + + /*! + this method looks for the smallest set bit + + result: + if 'this' is not zero: + return value - true + 'table_id' - the index of a word <0..value_size-1> + 'index' - the index of this set bit in the word <0..TTMATH_BITS_PER_UINT) + + if 'this' is zero: + return value - false + both 'table_id' and 'index' are zero + */ + bool FindLowestBit(uint & table_id, uint & index) const + { + for(table_id=0 ; table_id= value_size ) + { + // is zero + index = 0; + table_id = 0; + + return false; + } + + // table[table_id] is different from 0 + index = FindLowestBitInWord( table[table_id] ); + + return true; + } + + + /*! + getting the 'bit_index' bit + + bit_index bigger or equal zero + */ + uint GetBit(uint bit_index) const + { + TTMATH_ASSERT( bit_index < value_size * TTMATH_BITS_PER_UINT ) + + uint index = bit_index / TTMATH_BITS_PER_UINT; + uint bit = bit_index % TTMATH_BITS_PER_UINT; + + uint temp = table[index]; + uint res = SetBitInWord(temp, bit); + + return res; + } + + + /*! + setting the 'bit_index' bit + and returning the last state of the bit + + bit_index bigger or equal zero + */ + uint SetBit(uint bit_index) + { + TTMATH_ASSERT( bit_index < value_size * TTMATH_BITS_PER_UINT ) + + uint index = bit_index / TTMATH_BITS_PER_UINT; + uint bit = bit_index % TTMATH_BITS_PER_UINT; + uint res = SetBitInWord(table[index], bit); + + TTMATH_LOG("UInt::SetBit") + + return res; + } + + + /*! + this method performs a bitwise operation AND + */ + void BitAnd(const UInt & ss2) + { + for(uint x=0 ; x & ss2) + { + for(uint x=0 ; x & ss2) + { + for(uint x=0 ; x + + for example: + BitNot2(8) = BitNot2( 1000(bin) ) = 111(bin) = 7 + */ + void BitNot2() + { + uint table_id, index; + + if( FindLeadingBit(table_id, index) ) + { + for(uint x=0 ; x>= shift; + + table[table_id] ^= mask; + } + else + table[0] = 1; + + + TTMATH_LOG("UInt::BitNot2") + } + + + + /*! + * + * Multiplication + * + * + */ + +public: + + /*! + multiplication: this = this * ss2 + + it can return a carry + */ + uint MulInt(uint ss2) + { + uint r1, r2, x1; + uint c = 0; + + UInt u(*this); + SetZero(); + + if( ss2 == 0 ) + { + TTMATH_LOGC("UInt::MulInt(uint)", 0) + return 0; + } + + for(x1=0 ; x1 + void MulInt(uint ss2, UInt & result) const + { + TTMATH_ASSERT( result_size > value_size ) + + uint r2,r1; + uint x1size=value_size; + uint x1start=0; + + result.SetZero(); + + if( ss2 == 0 ) + { + TTMATH_VECTOR_LOG("UInt::MulInt(uint, UInt<>)", result.table, result_size) + return; + } + + if( value_size > 2 ) + { + // if the value_size is smaller than or equal to 2 + // there is no sense to set x1size and x1start to another values + + for(x1size=value_size ; x1size>0 && table[x1size-1]==0 ; --x1size); + + if( x1size == 0 ) + { + TTMATH_VECTOR_LOG("UInt::MulInt(uint, UInt<>)", result.table, result_size) + return; + } + + for(x1start=0 ; x1start)", result.table, result_size) + + return; + } + + + + /*! + the multiplication 'this' = 'this' * ss2 + + algorithm: 100 - means automatically choose the fastest algorithm + */ + uint Mul(const UInt & ss2, uint algorithm = 100) + { + switch( algorithm ) + { + case 1: + return Mul1(ss2); + + case 2: + return Mul2(ss2); + + case 3: + return Mul3(ss2); + + case 100: + default: + return MulFastest(ss2); + } + } + + + /*! + the multiplication 'result' = 'this' * ss2 + + since the 'result' is twice bigger than 'this' and 'ss2' + this method never returns a carry + + algorithm: 100 - means automatically choose the fastest algorithm + */ + void MulBig(const UInt & ss2, + UInt & result, + uint algorithm = 100) + { + switch( algorithm ) + { + case 1: + return Mul1Big(ss2, result); + + case 2: + return Mul2Big(ss2, result); + + case 3: + return Mul3Big(ss2, result); + + case 100: + default: + return MulFastestBig(ss2, result); + } + } + + + + /*! + the first version of the multiplication algorithm + */ + +private: + + /*! + multiplication: this = this * ss2 + + it returns carry if it has been + */ + uint Mul1Ref(const UInt & ss2) + { + TTMATH_REFERENCE_ASSERT( ss2 ) + + UInt ss1( *this ); + SetZero(); + + for(uint i=0; i < value_size*TTMATH_BITS_PER_UINT ; ++i) + { + if( Add(*this) ) + { + TTMATH_LOGC("UInt::Mul1", 1) + return 1; + } + + if( ss1.Rcl(1) ) + if( Add(ss2) ) + { + TTMATH_LOGC("UInt::Mul1", 1) + return 1; + } + } + + TTMATH_LOGC("UInt::Mul1", 0) + + return 0; + } + + +public: + + /*! + multiplication: this = this * ss2 + can return carry + */ + uint Mul1(const UInt & ss2) + { + if( this == &ss2 ) + { + UInt copy_ss2(ss2); + return Mul1Ref(copy_ss2); + } + else + { + return Mul1Ref(ss2); + } + } + + + /*! + multiplication: result = this * ss2 + + result is twice bigger than 'this' and 'ss2' + this method never returns carry + */ + void Mul1Big(const UInt & ss2_, UInt & result) + { + UInt ss2; + uint i; + + // copying *this into result and ss2_ into ss2 + for(i=0 ; i & ss2) + { + UInt result; + uint i, c = 0; + + Mul2Big(ss2, result); + + // copying result + for(i=0 ; i & ss2, UInt & result) + { + Mul2Big2(table, ss2.table, result); + + TTMATH_LOG("UInt::Mul2Big") + } + + +private: + + /*! + an auxiliary method for calculating the multiplication + + arguments we're taking as pointers (this is to improve the Mul3Big2()- avoiding + unnecessary copying objects), the result should be taken as a pointer too, + but at the moment there is no method AddTwoInts() which can operate on pointers + */ + template + void Mul2Big2(const uint * ss1, const uint * ss2, UInt & result) + { + uint x1size = ss_size, x2size = ss_size; + uint x1start = 0, x2start = 0; + + if( ss_size > 2 ) + { + // if the ss_size is smaller than or equal to 2 + // there is no sense to set x1size (and others) to another values + + for(x1size=ss_size ; x1size>0 && ss1[x1size-1]==0 ; --x1size); + for(x2size=ss_size ; x2size>0 && ss2[x2size-1]==0 ; --x2size); + + for(x1start=0 ; x1start(ss1, ss2, result, x1start, x1size, x2start, x2size); + } + + + + /*! + an auxiliary method for calculating the multiplication + */ + template + void Mul2Big3(const uint * ss1, const uint * ss2, UInt & result, uint x1start, uint x1size, uint x2start, uint x2size) + { + uint r2, r1; + + result.SetZero(); + + if( x1size==0 || x2size==0 ) + return; + + for(uint x1=x1start ; x1 & ss2) + { + UInt result; + uint i, c = 0; + + Mul3Big(ss2, result); + + // copying result + for(i=0 ; i & ss2, UInt & result) + { + Mul3Big2(table, ss2.table, result.table); + + TTMATH_LOG("UInt::Mul3Big") + } + + + +private: + + /*! + an auxiliary method for calculating the Karatsuba multiplication + + result_size is equal ss_size*2 + */ + template + void Mul3Big2(const uint * ss1, const uint * ss2, uint * result) + { + const uint * x1, * x0, * y1, * y0; + + + if( ss_size>1 && ss_size res; + Mul2Big2(ss1, ss2, res); + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wtautological-compare" +#endif + + for(uint i=0 ; i(x1, x0, y1, y0, result); + } + else + { + // ss_size is even + x0 = ss1; + y0 = ss2; + x1 = ss1 + ss_size / 2; + y1 = ss2 + ss_size / 2; + + // all four vectors (x0 x1 y0 y1) are equal in size + Mul3Big3(x1, x0, y1, y0, result); + } + } + + + +#ifdef _MSC_VER +#pragma warning (disable : 4717) +//warning C4717: recursive on all control paths, function will cause runtime stack overflow +//we have the stop point in Mul3Big2() method +#endif + + + /*! + an auxiliary method for calculating the Karatsuba multiplication + + x = x1*B^m + x0 + y = y1*B^m + y0 + + first_size - is the size of vectors: x0 and y0 + second_size - is the size of vectors: x1 and y1 (can be either equal first_size or smaller about one from first_size) + + x*y = (x1*B^m + x0)(y1*B^m + y0) = z2*B^(2m) + z1*B^m + z0 + where + z0 = x0*y0 + z2 = x1*y1 + z1 = (x1 + x0)*(y1 + y0) - z2 - z0 + */ + template + uint Mul3Big3(const uint * x1, const uint * x0, const uint * y1, const uint * y0, uint * result) + { + uint i, c, xc, yc; + + UInt temp, temp2; + UInt z1; + + // z0 and z2 we store directly in the result (we don't use any temporary variables) + Mul3Big2(x0, y0, result); // z0 + Mul3Big2(x1, y1, result+first_size*2); // z2 + + // now we calculate z1 + // temp = (x0 + x1) + // temp2 = (y0 + y1) + // we're using temp and temp2 with UInt, although there can be a carry but + // we simple remember it in xc and yc (xc and yc can be either 0 or 1), + // and (x0 + x1)*(y0 + y1) we calculate in this way (schoolbook algorithm): + // + // xc | temp + // yc | temp2 + // -------------------- + // (temp * temp2) + // xc*temp2 | + // yc*temp | + // xc*yc | + // ---------- z1 -------- + // + // and the result is never larger in size than 3*first_size + + xc = AddVector(x0, x1, first_size, second_size, temp.table); + yc = AddVector(y0, y1, first_size, second_size, temp2.table); + + Mul3Big2(temp.table, temp2.table, z1.table); + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wtautological-compare" +#endif + + // clearing the rest of z1 + for(i=first_size*2 ; i second_size ) + { + uint z1_size = result_size - first_size; + TTMATH_ASSERT( z1_size <= first_size*3 ) + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wtautological-compare" +#endif + + for(i=z1_size ; i & ss2) + { + UInt result; + uint i, c = 0; + + MulFastestBig(ss2, result); + + // copying result + for(i=0 ; i & ss2, UInt & result) + { + if( value_size < TTMATH_USE_KARATSUBA_MULTIPLICATION_FROM_SIZE ) + return Mul2Big(ss2, result); + + uint x1size = value_size, x2size = value_size; + uint x1start = 0, x2start = 0; + + for(x1size=value_size ; x1size>0 && table[x1size-1]==0 ; --x1size); + for(x2size=value_size ; x2size>0 && ss2.table[x2size-1]==0 ; --x2size); + + if( x1size==0 || x2size==0 ) + { + // either 'this' or 'ss2' is equal zero - the result is zero too + result.SetZero(); + return; + } + + for(x1start=0 ; x1start(table, ss2.table, result, x1start, x1size, x2start, x2size); + + + // Karatsuba multiplication + Mul3Big(ss2, result); + + TTMATH_LOG("UInt::MulFastestBig") + } + + + /*! + * + * Division + * + * + */ + +public: + + + /*! + division by one unsigned word + + returns 1 when divisor is zero + */ + uint DivInt(uint divisor, uint * remainder = 0) + { + if( divisor == 0 ) + { + if( remainder ) + *remainder = 0; // this is for convenience, without it the compiler can report that 'remainder' is uninitialized + + TTMATH_LOG("UInt::DivInt") + + return 1; + } + + if( divisor == 1 ) + { + if( remainder ) + *remainder = 0; + + TTMATH_LOG("UInt::DivInt") + + return 0; + } + + UInt dividend(*this); + SetZero(); + + sint i; // i must be with a sign + uint r = 0; + + // we're looking for the last word in ss1 + for(i=value_size-1 ; i>0 && dividend.table[i]==0 ; --i); + + for( ; i>=0 ; --i) + DivTwoWords(r, dividend.table[i], divisor, &table[i], &r); + + if( remainder ) + *remainder = r; + + TTMATH_LOG("UInt::DivInt") + + return 0; + } + + uint DivInt(uint divisor, uint & remainder) + { + return DivInt(divisor, &remainder); + } + + + + /*! + division this = this / ss2 + + return values: + 0 - ok + 1 - division by zero + 'this' will be the quotient + 'remainder' - remainder + */ + uint Div( const UInt & divisor, + UInt * remainder = 0, + uint algorithm = 3) + { + switch( algorithm ) + { + case 1: + return Div1(divisor, remainder); + + case 2: + return Div2(divisor, remainder); + + case 3: + default: + return Div3(divisor, remainder); + } + } + + uint Div(const UInt & divisor, UInt & remainder, uint algorithm = 3) + { + return Div(divisor, &remainder, algorithm); + } + + + +private: + + /*! + return values: + 0 - none has to be done + 1 - division by zero + 2 - division should be made + */ + uint Div_StandardTest( const UInt & v, + uint & m, uint & n, + UInt * remainder = 0) + { + switch( Div_CalculatingSize(v, m, n) ) + { + case 4: // 'this' is equal v + if( remainder ) + remainder->SetZero(); + + SetOne(); + TTMATH_LOG("UInt::Div_StandardTest") + return 0; + + case 3: // 'this' is smaller than v + if( remainder ) + *remainder = *this; + + SetZero(); + TTMATH_LOG("UInt::Div_StandardTest") + return 0; + + case 2: // 'this' is zero + if( remainder ) + remainder->SetZero(); + + SetZero(); + TTMATH_LOG("UInt::Div_StandardTest") + return 0; + + case 1: // v is zero + TTMATH_LOG("UInt::Div_StandardTest") + return 1; + } + + TTMATH_LOG("UInt::Div_StandardTest") + + return 2; + } + + + + /*! + return values: + 0 - ok + 'm' - is the index (from 0) of last non-zero word in table ('this') + 'n' - is the index (from 0) of last non-zero word in v.table + 1 - v is zero + 2 - 'this' is zero + 3 - 'this' is smaller than v + 4 - 'this' is equal v + + if the return value is different than zero the 'm' and 'n' are undefined + */ + uint Div_CalculatingSize(const UInt & v, uint & m, uint & n) + { + m = n = value_size-1; + + for( ; n!=0 && v.table[n]==0 ; --n); + + if( n==0 && v.table[n]==0 ) + return 1; + + for( ; m!=0 && table[m]==0 ; --m); + + if( m==0 && table[m]==0 ) + return 2; + + if( m < n ) + return 3; + else + if( m == n ) + { + uint i; + for(i = n ; i!=0 && table[i]==v.table[i] ; --i); + + if( table[i] < v.table[i] ) + return 3; + else + if (table[i] == v.table[i] ) + return 4; + } + + return 0; + } + + +public: + + /*! + the first division algorithm + radix 2 + */ + uint Div1(const UInt & divisor, UInt * remainder = 0) + { + uint m,n, test; + + test = Div_StandardTest(divisor, m, n, remainder); + if( test < 2 ) + return test; + + if( !remainder ) + { + UInt rem; + + return Div1_Calculate(divisor, rem); + } + + return Div1_Calculate(divisor, *remainder); + } + + + /*! + the first division algorithm + radix 2 + */ + uint Div1(const UInt & divisor, UInt & remainder) + { + return Div1(divisor, &remainder); + } + + +private: + + uint Div1_Calculate(const UInt & divisor, UInt & rest) + { + if( this == &divisor ) + { + UInt divisor_copy(divisor); + return Div1_CalculateRef(divisor_copy, rest); + } + else + { + return Div1_CalculateRef(divisor, rest); + } + } + + + uint Div1_CalculateRef(const UInt & divisor, UInt & rest) + { + TTMATH_REFERENCE_ASSERT( divisor ) + + sint loop; + sint c; + + rest.SetZero(); + loop = value_size * TTMATH_BITS_PER_UINT; + c = 0; + + + div_a: + c = Rcl(1, c); + c = rest.Add(rest,c); + c = rest.Sub(divisor,c); + + c = !c; + + if(!c) + goto div_d; + + + div_b: + --loop; + if(loop) + goto div_a; + + c = Rcl(1, c); + TTMATH_LOG("UInt::Div1_Calculate") + return 0; + + + div_c: + c = Rcl(1, c); + c = rest.Add(rest,c); + c = rest.Add(divisor); + + if(c) + goto div_b; + + + div_d: + --loop; + if(loop) + goto div_c; + + c = Rcl(1, c); + c = rest.Add(divisor); + + TTMATH_LOG("UInt::Div1_Calculate") + + return 0; + } + + +public: + + /*! + the second division algorithm + + return values: + 0 - ok + 1 - division by zero + */ + uint Div2(const UInt & divisor, UInt * remainder = 0) + { + if( this == &divisor ) + { + UInt divisor_copy(divisor); + return Div2Ref(divisor_copy, remainder); + } + else + { + return Div2Ref(divisor, remainder); + } + } + + + /*! + the second division algorithm + + return values: + 0 - ok + 1 - division by zero + */ + uint Div2(const UInt & divisor, UInt & remainder) + { + return Div2(divisor, &remainder); + } + + +private: + + /*! + the second division algorithm + + return values: + 0 - ok + 1 - division by zero + */ + uint Div2Ref(const UInt & divisor, UInt * remainder = 0) + { + uint bits_diff; + uint status = Div2_Calculate(divisor, remainder, bits_diff); + if( status < 2 ) + return status; + + if( CmpBiggerEqual(divisor) ) + { + Div2(divisor, remainder); + SetBit(bits_diff); + } + else + { + if( remainder ) + *remainder = *this; + + SetZero(); + SetBit(bits_diff); + } + + TTMATH_LOG("UInt::Div2") + + return 0; + } + + + /*! + return values: + 0 - we've calculated the division + 1 - division by zero + 2 - we have to still calculate + + */ + uint Div2_Calculate(const UInt & divisor, UInt * remainder, + uint & bits_diff) + { + uint table_id, index; + uint divisor_table_id, divisor_index; + + uint status = Div2_FindLeadingBitsAndCheck( divisor, remainder, + table_id, index, + divisor_table_id, divisor_index); + + if( status < 2 ) + { + TTMATH_LOG("UInt::Div2_Calculate") + return status; + } + + // here we know that 'this' is greater than divisor + // then 'index' is greater or equal 'divisor_index' + bits_diff = index - divisor_index; + + UInt divisor_copy(divisor); + divisor_copy.Rcl(bits_diff, 0); + + if( CmpSmaller(divisor_copy, table_id) ) + { + divisor_copy.Rcr(1); + --bits_diff; + } + + Sub(divisor_copy, 0); + + TTMATH_LOG("UInt::Div2_Calculate") + + return 2; + } + + + /*! + return values: + 0 - we've calculated the division + 1 - division by zero + 2 - we have to still calculate + */ + uint Div2_FindLeadingBitsAndCheck( const UInt & divisor, + UInt * remainder, + uint & table_id, uint & index, + uint & divisor_table_id, uint & divisor_index) + { + if( !divisor.FindLeadingBit(divisor_table_id, divisor_index) ) + { + // division by zero + TTMATH_LOG("UInt::Div2_FindLeadingBitsAndCheck") + return 1; + } + + if( !FindLeadingBit(table_id, index) ) + { + // zero is divided by something + + SetZero(); + + if( remainder ) + remainder->SetZero(); + + TTMATH_LOG("UInt::Div2_FindLeadingBitsAndCheck") + + return 0; + } + + divisor_index += divisor_table_id * TTMATH_BITS_PER_UINT; + index += table_id * TTMATH_BITS_PER_UINT; + + if( divisor_table_id == 0 ) + { + // dividor has only one 32-bit word + + uint r; + DivInt(divisor.table[0], &r); + + if( remainder ) + { + remainder->SetZero(); + remainder->table[0] = r; + } + + TTMATH_LOG("UInt::Div2_FindLeadingBitsAndCheck") + + return 0; + } + + + if( Div2_DivisorGreaterOrEqual( divisor, remainder, + table_id, index, + divisor_index) ) + { + TTMATH_LOG("UInt::Div2_FindLeadingBitsAndCheck") + return 0; + } + + + TTMATH_LOG("UInt::Div2_FindLeadingBitsAndCheck") + + return 2; + } + + + /*! + return values: + true if divisor is equal or greater than 'this' + */ + bool Div2_DivisorGreaterOrEqual( const UInt & divisor, + UInt * remainder, + uint table_id, uint index, + uint divisor_index ) + { + if( divisor_index > index ) + { + // divisor is greater than this + + if( remainder ) + *remainder = *this; + + SetZero(); + + TTMATH_LOG("UInt::Div2_DivisorGreaterOrEqual") + + return true; + } + + if( divisor_index == index ) + { + // table_id == divisor_table_id as well + + uint i; + for(i = table_id ; i!=0 && table[i]==divisor.table[i] ; --i); + + if( table[i] < divisor.table[i] ) + { + // divisor is greater than 'this' + + if( remainder ) + *remainder = *this; + + SetZero(); + + TTMATH_LOG("UInt::Div2_DivisorGreaterOrEqual") + + return true; + } + else + if( table[i] == divisor.table[i] ) + { + // divisor is equal 'this' + + if( remainder ) + remainder->SetZero(); + + SetOne(); + + TTMATH_LOG("UInt::Div2_DivisorGreaterOrEqual") + + return true; + } + } + + TTMATH_LOG("UInt::Div2_DivisorGreaterOrEqual") + + return false; + } + + +public: + + /*! + the third division algorithm + */ + uint Div3(const UInt & ss2, UInt * remainder = 0) + { + if( this == &ss2 ) + { + UInt copy_ss2(ss2); + return Div3Ref(copy_ss2, remainder); + } + else + { + return Div3Ref(ss2, remainder); + } + } + + + /*! + the third division algorithm + */ + uint Div3(const UInt & ss2, UInt & remainder) + { + return Div3(ss2, &remainder); + } + + +private: + + /*! + the third division algorithm + + this algorithm is described in the following book: + "The art of computer programming 2" (4.3.1 page 272) + Donald E. Knuth + !! give the description here (from the book) + */ + uint Div3Ref(const UInt & v, UInt * remainder = 0) + { + uint m,n, test; + + test = Div_StandardTest(v, m, n, remainder); + if( test < 2 ) + return test; + + if( n == 0 ) + { + uint r; + DivInt( v.table[0], &r ); + + if( remainder ) + { + remainder->SetZero(); + remainder->table[0] = r; + } + + TTMATH_LOG("UInt::Div3") + + return 0; + } + + + // we can only use the third division algorithm when + // the divisor is greater or equal 2^32 (has more than one 32-bit word) + ++m; + ++n; + m = m - n; + Div3_Division(v, remainder, m, n); + + TTMATH_LOG("UInt::Div3") + + return 0; + } + + + +private: + + + void Div3_Division(UInt v, UInt * remainder, uint m, uint n) + { + TTMATH_ASSERT( n>=2 ) + + UInt uu, vv; + UInt q; + uint d, u_value_size, u0, u1, u2, v1, v0, j=m; + + u_value_size = Div3_Normalize(v, n, d); + + if( j+n == value_size ) + u2 = u_value_size; + else + u2 = table[j+n]; + + Div3_MakeBiggerV(v, vv); + + for(uint i = j+1 ; i & uu, uint j, uint n, uint u_max) + { + uint i; + + for(i=0 ; i so and 'i' is from <0..value_size> + // then table[i] is always correct (look at the declaration of 'uu') + uu.table[i] = u_max; + + for( ++i ; i & uu, uint j, uint n) + { + uint i; + + for(i=0 ; i & v, UInt & vv) + { + for(uint i=0 ; i & v, uint n, uint & d) + { + // v.table[n-1] is != 0 + + uint bit = (uint)FindLeadingBitInWord(v.table[n-1]); + uint move = (TTMATH_BITS_PER_UINT - bit - 1); + uint res = table[value_size-1]; + d = move; + + if( move > 0 ) + { + v.Rcl(move, 0); + Rcl(move, 0); + res = res >> (bit + 1); + } + else + { + res = 0; + } + + TTMATH_LOG("UInt::Div3_Normalize") + + return res; + } + + + void Div3_Unnormalize(UInt * remainder, uint n, uint d) + { + for(uint i=n ; i u_temp; + uint rp; + bool next_test; + + TTMATH_ASSERT( v1 != 0 ) + + u_temp.table[1] = u2; + u_temp.table[0] = u1; + u_temp.DivInt(v1, &rp); + + TTMATH_ASSERT( u_temp.table[1]==0 || u_temp.table[1]==1 ) + + do + { + bool decrease = false; + + if( u_temp.table[1] == 1 ) + decrease = true; + else + { + UInt<2> temp1, temp2; + + UInt<2>::MulTwoWords(u_temp.table[0], v0, temp1.table+1, temp1.table); + temp2.table[1] = rp; + temp2.table[0] = u0; + + if( temp1 > temp2 ) + decrease = true; + } + + next_test = false; + + if( decrease ) + { + u_temp.SubOne(); + + rp += v1; + + if( rp >= v1 ) // it means that there wasn't a carry (r & uu, + const UInt & vv, uint & qp) + { + // D4 (in the book) + + UInt vv_temp(vv); + vv_temp.MulInt(qp); + + if( uu.Sub(vv_temp) ) + { + // there was a carry + + // + // !!! this part of code was not tested + // + + --qp; + uu.Add(vv); + + // can be a carry from this additions but it should be ignored + // because it cancels with the borrow from uu.Sub(vv_temp) + } + + TTMATH_LOG("UInt::Div3_MultiplySubtract") + } + + + + + + +public: + + + /*! + power this = this ^ pow + binary algorithm (r-to-l) + + return values: + 0 - ok + 1 - carry + 2 - incorrect argument (0^0) + */ + uint Pow(UInt pow) + { + if(pow.IsZero() && IsZero()) + // we don't define zero^zero + return 2; + + UInt start(*this); + UInt result; + result.SetOne(); + uint c = 0; + + while( !c ) + { + if( pow.table[0] & 1 ) + c += result.Mul(start); + + pow.Rcr2_one(0); + if( pow.IsZero() ) + break; + + c += start.Mul(start); + } + + *this = result; + + TTMATH_LOGC("UInt::Pow(UInt<>)", c) + + return (c==0)? 0 : 1; + } + + + /*! + square root + e.g. Sqrt(9) = 3 + ('digit-by-digit' algorithm) + */ + void Sqrt() + { + UInt bit, temp; + + if( IsZero() ) + return; + + UInt value(*this); + + SetZero(); + bit.SetZero(); + bit.table[value_size-1] = (TTMATH_UINT_HIGHEST_BIT >> 1); + + while( bit > value ) + bit.Rcr(2); + + while( !bit.IsZero() ) + { + temp = *this; + temp.Add(bit); + + if( value >= temp ) + { + value.Sub(temp); + Rcr(1); + Add(bit); + } + else + { + Rcr(1); + } + + bit.Rcr(2); + } + + TTMATH_LOG("UInt::Sqrt") + } + + + + /*! + this method sets n first bits to value zero + + For example: + let n=2 then if there's a value 111 (bin) there'll be '100' (bin) + */ + void ClearFirstBits(uint n) + { + if( n >= value_size*TTMATH_BITS_PER_UINT ) + { + SetZero(); + TTMATH_LOG("UInt::ClearFirstBits") + return; + } + + uint * p = table; + + // first we're clearing the whole words + while( n >= TTMATH_BITS_PER_UINT ) + { + *p++ = 0; + n -= TTMATH_BITS_PER_UINT; + } + + if( n == 0 ) + { + TTMATH_LOG("UInt::ClearFirstBits") + return; + } + + // and then we're clearing one word which has left + // mask -- all bits are set to one + uint mask = TTMATH_UINT_MAX_VALUE; + + mask = mask << n; + + (*p) &= mask; + + TTMATH_LOG("UInt::ClearFirstBits") + } + + + /*! + this method returns true if the highest bit of the value is set + */ + bool IsTheHighestBitSet() const + { + return (table[value_size-1] & TTMATH_UINT_HIGHEST_BIT) != 0; + } + + + /*! + this method returns true if the lowest bit of the value is set + */ + bool IsTheLowestBitSet() const + { + return (*table & 1) != 0; + } + + + /*! + returning true if only the highest bit is set + */ + bool IsOnlyTheHighestBitSet() const + { +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wtautological-compare" +#endif + + for(uint i=0 ; i> (TTMATH_BITS_PER_UINT - rest); + + return (table[i] & mask) == 0; + } + + + + /*! + * + * conversion methods + * + */ + + + + /*! + this method converts an UInt type to this class + + this operation has mainly sense if the value from p is + equal or smaller than that one which is returned from UInt::SetMax() + + it returns a carry if the value 'p' is too big + */ + template + uint FromUInt(const UInt & p) + { + uint min_size = (value_size < argument_size)? value_size : argument_size; + uint i; + + for(i=0 ; i argument_size ) + { + // 'this' is longer than 'p' + + for( ; i)", 1) + return 1; + } + } + + TTMATH_LOGC("UInt::FromUInt(UInt<>)", 0) + + return 0; + } + + + /*! + this method converts an UInt type to this class + + this operation has mainly sense if the value from p is + equal or smaller than that one which is returned from UInt::SetMax() + + it returns a carry if the value 'p' is too big + */ + template + uint FromInt(const UInt & p) + { + return FromUInt(p); + } + + + /*! + this method converts the uint type to this class + */ + uint FromUInt(uint value) + { + for(uint i=1 ; i type to this class + + it doesn't return a carry + */ +/* template + UInt & operator=(const UInt & p) + { + FromUInt(p); + + return *this; + } +*/ + + /*! + the assignment operator + */ +/* UInt & operator=(const UInt & p) + { + for(uint i=0 ; i)") + + return *this; + } +*/ + + /*! + this method converts the uint type to this class + */ + UInt & operator=(uint i) + { + FromUInt(i); + + return *this; + } + + + /*! + a constructor for converting the uint to this class + */ +/* UInt(uint i) + { + FromUInt(i); + } +*/ + + /*! + this method converts the sint type to this class + */ + UInt & operator=(sint i) + { + FromInt(i); + + return *this; + } + + + /*! + a constructor for converting the sint to this class + + look at the description of UInt::operator=(sint) + */ +/* UInt(sint i) + { + FromInt(i); + } +*/ + +#ifdef TTMATH_PLATFORM32 + + + /*! + this method converts unsigned 64 bit int type to this class + ***this method is created only on a 32bit platform*** + */ + uint FromUInt(ulint n) + { + table[0] = (uint)n; + + if( value_size == 1 ) + { + uint c = ((n >> TTMATH_BITS_PER_UINT) == 0) ? 0 : 1; + + TTMATH_LOGC("UInt::FromUInt(ulint)", c) + return c; + } + + table[1] = (uint)(n >> TTMATH_BITS_PER_UINT); + + for(uint i=2 ; i & operator=(ulint n) + { + FromUInt(n); + + return *this; + } + + + /*! + a constructor for converting unsigned 64 bit int to this class + ***this constructor is created only on a 32bit platform*** + */ +/* UInt(ulint n) + { + FromUInt(n); + } +*/ + + /*! + this operator converts signed 64 bit int type to this class + ***this operator is created only on a 32bit platform*** + */ + UInt & operator=(slint n) + { + FromInt(n); + + return *this; + } + + + /*! + a constructor for converting signed 64 bit int to this class + ***this constructor is created only on a 32bit platform*** + */ +/* UInt(slint n) + { + FromInt(n); + } +*/ +#endif + + + +#ifdef TTMATH_PLATFORM64 + + + /*! + this method converts 32 bit unsigned int type to this class + ***this operator is created only on a 64bit platform*** + */ + uint FromUInt(unsigned int i) + { + return FromUInt(uint(i)); + } + + /*! + this method converts 32 bit unsigned int type to this class + ***this operator is created only on a 64bit platform*** + */ + uint FromInt(unsigned int i) + { + return FromUInt(uint(i)); + } + + + /*! + this method converts 32 bit signed int type to this class + ***this operator is created only on a 64bit platform*** + */ + uint FromInt(signed int i) + { + return FromInt(sint(i)); + } + + + /*! + this operator converts 32 bit unsigned int type to this class + ***this operator is created only on a 64bit platform*** + */ + UInt & operator=(unsigned int i) + { + FromUInt(i); + + return *this; + } + + + /*! + a constructor for converting 32 bit unsigned int to this class + ***this constructor is created only on a 64bit platform*** + */ +/* UInt(unsigned int i) + { + FromUInt(i); + } +*/ + + /*! + an operator for converting 32 bit signed int to this class + ***this constructor is created only on a 64bit platform*** + */ + UInt & operator=(signed int i) + { + FromInt(i); + + return *this; + } + + + /*! + a constructor for converting 32 bit signed int to this class + ***this constructor is created only on a 64bit platform*** + */ +/* UInt(signed int i) + { + FromInt(i); + } +*/ + +#endif + + + + + + /*! + a constructor for converting a string to this class (with the base=10) + */ +/* UInt(const char * s) + { + FromString(s); + } +*/ + + /*! + a constructor for converting a string to this class (with the base=10) + */ +/* UInt(const std::string & s) + { + FromString( s.c_str() ); + } +*/ + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + a constructor for converting a string to this class (with the base=10) + */ + UInt(const wchar_t * s) + { + FromString(s); + } + + + /*! + a constructor for converting a string to this class (with the base=10) + */ + UInt(const std::wstring & s) + { + FromString( s.c_str() ); + } + +#endif + + + + + /*! + a default constructor + + we don't clear the table + */ +/* UInt() + { + // when macro TTMATH_DEBUG_LOG is defined + // we set special values to the table + // in order to be everywhere the same value of the UInt object + // without this it would be difficult to analyse the log file + #ifdef TTMATH_DEBUG_LOG + #ifdef TTMATH_PLATFORM32 + for(uint i=0 ; i & u) + { + for(uint i=0 ; i)") + } +*/ + + + /*! + a template for producting constructors for copying from another types + */ +/* template + UInt(const UInt & u) + { + // look that 'size' we still set as 'value_size' and not as u.value_size + FromUInt(u); + } +*/ + + + + /*! + a destructor + */ +/* ~UInt() + { + } +*/ + + /*! + this method returns the lowest value from table + + we must be sure when we using this method whether the value + will hold in an uint type or not (the rest value from the table must be zero) + */ + uint ToUInt() const + { + return table[0]; + } + + + /*! + this method converts the value to uint type + can return a carry if the value is too long to store it in uint type + */ + uint ToUInt(uint & result) const + { + result = table[0]; + + for(uint i=1 ; i> 32) != 0 ) + return 1; + + for(uint i=1 ; i + */ + double ToStringLog2(uint x) const + { + static double log_tab[] = { + 1.000000000000000000, + 0.630929753571457437, + 0.500000000000000000, + 0.430676558073393050, + 0.386852807234541586, + 0.356207187108022176, + 0.333333333333333333, + 0.315464876785728718, + 0.301029995663981195, + 0.289064826317887859, + 0.278942945651129843, + 0.270238154427319741, + 0.262649535037193547, + 0.255958024809815489, + 0.250000000000000000 + }; + + if( x<2 || x>16 ) + return 0; + + return log_tab[x-2]; + } + + +public: + + + /*! + an auxiliary method for converting to a string + it's used from Int::ToString() too (negative is set true then) + */ + template + void ToStringBase(string_type & result, uint b = 10, bool negative = false) const + { + UInt temp(*this); + uint rest, table_id, index, digits; + double digits_d; + char character; + + result.clear(); + + if( b<2 || b>16 ) + return; + + if( !FindLeadingBit(table_id, index) ) + { + result = '0'; + return; + } + + if( negative ) + result = '-'; + + digits_d = table_id; // for not making an overflow in uint type + digits_d *= TTMATH_BITS_PER_UINT; + digits_d += index + 1; + digits_d *= ToStringLog2(b); + digits = static_cast(digits_d) + 3; // plus some epsilon + + if( result.capacity() < digits ) + result.reserve(digits); + + do + { + temp.DivInt(b, &rest); + character = static_cast(Misc::DigitToChar(rest)); + result.insert(result.end(), character); + } + while( !temp.IsZero() ); + + size_t i1 = negative ? 1 : 0; // the first is a hyphen (when negative is true) + size_t i2 = result.size() - 1; + + for( ; i1 < i2 ; ++i1, --i2 ) + { + char tempc = static_cast(result[i1]); + result[i1] = result[i2]; + result[i2] = tempc; + } + } + + + + /*! + this method converts the value to a string with a base equal 'b' + */ + void ToString(std::string & result, uint b = 10) const + { + return ToStringBase(result, b); + } + + + std::string ToString(uint b = 10) const + { + std::string result; + ToStringBase(result, b); + + return result; + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + void ToString(std::wstring & result, uint b = 10) const + { + return ToStringBase(result, b); + } + + std::wstring ToWString(uint b = 10) const + { + std::wstring result; + ToStringBase(result, b); + + return result; + } + +#endif + + + +private: + + /*! + an auxiliary method for converting from a string + */ + template + uint FromStringBase(const char_type * s, uint b = 10, const char_type ** after_source = 0, bool * value_read = 0) + { + UInt base; + base.FromUInt( b ); + UInt temp; + sint z; + uint c = 0; + + SetZero(); + temp.SetZero(); + Misc::SkipWhiteCharacters(s); + + if( after_source ) + *after_source = s; + + if( value_read ) + *value_read = false; + + if( b<2 || b>16 ) + return 1; + + + for( ; (z=Misc::CharToDigit(*s, b)) != -1 ; ++s) + { + if( value_read ) + *value_read = true; + + if( c == 0 ) + { + temp.table[0] = z; + + c += Mul(base); // !! IMPROVE ME: there can be used MulInt here + c += Add(temp); + } + } + + if( after_source ) + *after_source = s; + + TTMATH_LOGC("UInt::FromString", c) + + return (c==0)? 0 : 1; + } + + +public: + + + /*! + this method converts a string into its value + it returns carry=1 if the value will be too big or an incorrect base 'b' is given + + string is ended with a non-digit value, for example: + "12" will be translated to 12 + as well as: + "12foo" will be translated to 12 too + + existing first white characters will be ommited + + if the value from s is too large the rest digits will be skipped + + after_source (if exists) is pointing at the end of the parsed string + + value_read (if exists) tells whether something has actually been read (at least one digit) + */ + uint FromString(const char * s, uint b = 10, const char ** after_source = 0, bool * value_read = 0) + { + return FromStringBase(s, b, after_source, value_read); + } + + + /*! + this method converts a string into its value + + (it returns carry=1 if the value will be too big or an incorrect base 'b' is given) + */ + uint FromString(const std::string & s, uint b = 10) + { + return FromString( s.c_str(), b ); + } + + + /*! + this operator converts a string into its value (with base = 10) + */ + UInt & operator=(const char * s) + { + FromString(s); + + return *this; + } + + + /*! + this operator converts a string into its value (with base = 10) + */ + UInt & operator=(const std::string & s) + { + FromString( s.c_str() ); + + return *this; + } + + + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + this method converts a string into its value + */ + uint FromString(const wchar_t * s, uint b = 10, const wchar_t ** after_source = 0, bool * value_read = 0) + { + return FromStringBase(s, b, after_source, value_read); + } + + + /*! + this method converts a string into its value + + (it returns carry=1 if the value will be too big or an incorrect base 'b' is given) + */ + uint FromString(const std::wstring & s, uint b = 10) + { + return FromString( s.c_str(), b ); + } + + + /*! + this operator converts a string into its value (with base = 10) + */ + UInt & operator=(const wchar_t * s) + { + FromString(s); + + return *this; + } + + + /*! + this operator converts a string into its value (with base = 10) + */ + UInt & operator=(const std::wstring & s) + { + FromString( s.c_str() ); + + return *this; + } + +#endif + + + /*! + * + * methods for comparing + * + */ + + + /*! + this method returns true if 'this' is smaller than 'l' + + 'index' is an index of the first word from will be the comparison performed + (note: we start the comparison from back - from the last word, when index is -1 /default/ + it is automatically set into the last word) + I introduced it for some kind of optimization made in the second division algorithm (Div2) + */ + bool CmpSmaller(const UInt & l, sint index = -1) const + { + sint i; + + if( index==-1 || index>=sint(value_size) ) + i = value_size - 1; + else + i = index; + + + for( ; i>=0 ; --i) + { + if( table[i] != l.table[i] ) + return table[i] < l.table[i]; + } + + // they're equal + return false; + } + + + + /*! + this method returns true if 'this' is bigger than 'l' + + 'index' is an index of the first word from will be the comparison performed + (note: we start the comparison from back - from the last word, when index is -1 /default/ + it is automatically set into the last word) + + I introduced it for some kind of optimization made in the second division algorithm (Div2) + */ + bool CmpBigger(const UInt & l, sint index = -1) const + { + sint i; + + if( index==-1 || index>=sint(value_size) ) + i = value_size - 1; + else + i = index; + + + for( ; i>=0 ; --i) + { + if( table[i] != l.table[i] ) + return table[i] > l.table[i]; + } + + // they're equal + return false; + } + + + /*! + this method returns true if 'this' is equal 'l' + + 'index' is an index of the first word from will be the comparison performed + (note: we start the comparison from back - from the last word, when index is -1 /default/ + it is automatically set into the last word) + */ + bool CmpEqual(const UInt & l, sint index = -1) const + { + sint i; + + if( index==-1 || index>=sint(value_size) ) + i = value_size - 1; + else + i = index; + + + for( ; i>=0 ; --i) + if( table[i] != l.table[i] ) + return false; + + return true; + } + + + + /*! + this method returns true if 'this' is smaller than or equal 'l' + + 'index' is an index of the first word from will be the comparison performed + (note: we start the comparison from back - from the last word, when index is -1 /default/ + it is automatically set into the last word) + */ + bool CmpSmallerEqual(const UInt & l, sint index=-1) const + { + sint i; + + if( index==-1 || index>=sint(value_size) ) + i = value_size - 1; + else + i = index; + + + for( ; i>=0 ; --i) + { + if( table[i] != l.table[i] ) + return table[i] < l.table[i]; + } + + // they're equal + return true; + } + + + + /*! + this method returns true if 'this' is bigger than or equal 'l' + + 'index' is an index of the first word from will be the comparison performed + (note: we start the comparison from back - from the last word, when index is -1 /default/ + it is automatically set into the last word) + */ + bool CmpBiggerEqual(const UInt & l, sint index=-1) const + { + sint i; + + if( index==-1 || index>=sint(value_size) ) + i = value_size - 1; + else + i = index; + + + for( ; i>=0 ; --i) + { + if( table[i] != l.table[i] ) + return table[i] > l.table[i]; + } + + // they're equal + return true; + } + + + /* + operators for comparising + */ + + bool operator<(const UInt & l) const + { + return CmpSmaller(l); + } + + + bool operator>(const UInt & l) const + { + return CmpBigger(l); + } + + + bool operator==(const UInt & l) const + { + return CmpEqual(l); + } + + + bool operator!=(const UInt & l) const + { + return !operator==(l); + } + + + bool operator<=(const UInt & l) const + { + return CmpSmallerEqual(l); + } + + bool operator>=(const UInt & l) const + { + return CmpBiggerEqual(l); + } + + + /*! + * + * standard mathematical operators + * + */ + + UInt operator-(const UInt & p2) const + { + UInt temp(*this); + + temp.Sub(p2); + + return temp; + } + + UInt & operator-=(const UInt & p2) + { + Sub(p2); + + return *this; + } + + UInt operator+(const UInt & p2) const + { + UInt temp(*this); + + temp.Add(p2); + + return temp; + } + + UInt & operator+=(const UInt & p2) + { + Add(p2); + + return *this; + } + + + UInt operator*(const UInt & p2) const + { + UInt temp(*this); + + temp.Mul(p2); + + return temp; + } + + + UInt & operator*=(const UInt & p2) + { + Mul(p2); + + return *this; + } + + + UInt operator/(const UInt & p2) const + { + UInt temp(*this); + + temp.Div(p2); + + return temp; + } + + + UInt & operator/=(const UInt & p2) + { + Div(p2); + + return *this; + } + + + UInt operator%(const UInt & p2) const + { + UInt temp(*this); + UInt remainder; + + temp.Div( p2, remainder ); + + return remainder; + } + + + UInt & operator%=(const UInt & p2) + { + UInt remainder; + + Div( p2, remainder ); + operator=(remainder); + + return *this; + } + + + /*! + Prefix operator e.g ++variable + */ + UInt & operator++() + { + AddOne(); + + return *this; + } + + + /*! + Postfix operator e.g variable++ + */ + UInt operator++(int) + { + UInt temp( *this ); + + AddOne(); + + return temp; + } + + + UInt & operator--() + { + SubOne(); + + return *this; + } + + + UInt operator--(int) + { + UInt temp( *this ); + + SubOne(); + + return temp; + } + + + + /*! + * + * bitwise operators + * + */ + + UInt operator~() const + { + UInt temp( *this ); + + temp.BitNot(); + + return temp; + } + + + UInt operator&(const UInt & p2) const + { + UInt temp( *this ); + + temp.BitAnd(p2); + + return temp; + } + + + UInt & operator&=(const UInt & p2) + { + BitAnd(p2); + + return *this; + } + + + UInt operator|(const UInt & p2) const + { + UInt temp( *this ); + + temp.BitOr(p2); + + return temp; + } + + + UInt & operator|=(const UInt & p2) + { + BitOr(p2); + + return *this; + } + + + UInt operator^(const UInt & p2) const + { + UInt temp( *this ); + + temp.BitXor(p2); + + return temp; + } + + + UInt & operator^=(const UInt & p2) + { + BitXor(p2); + + return *this; + } + + + UInt operator>>(int move) const + { + UInt temp( *this ); + + temp.Rcr(move); + + return temp; + } + + + UInt & operator>>=(int move) + { + Rcr(move); + + return *this; + } + + + UInt operator<<(int move) const + { + UInt temp( *this ); + + temp.Rcl(move); + + return temp; + } + + + UInt & operator<<=(int move) + { + Rcl(move); + + return *this; + } + + + /*! + * + * input/output operators for standard streams + * + * (they are very simple, in the future they should be changed) + * + */ + + +private: + + + /*! + an auxiliary method for outputing to standard streams + */ + template + static ostream_type & OutputToStream(ostream_type & s, const UInt & l) + { + string_type ss; + + l.ToString(ss); + s << ss; + + return s; + } + + +public: + + + /*! + output to standard streams + */ + friend std::ostream & operator<<(std::ostream & s, const UInt & l) + { + return OutputToStream(s, l); + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + output to standard streams + */ + friend std::wostream & operator<<(std::wostream & s, const UInt & l) + { + return OutputToStream(s, l); + } + +#endif + + + +private: + + /*! + an auxiliary method for reading from standard streams + */ + template + static istream_type & InputFromStream(istream_type & s, UInt & l) + { + string_type ss; + + // char or wchar_t for operator>> + char_type z; + + // operator>> omits white characters if they're set for ommiting + s >> z; + + // we're reading only digits (base=10) + while( s.good() && Misc::CharToDigit(z, 10)>=0 ) + { + ss += z; + z = static_cast(s.get()); + } + + // we're leaving the last read character + // (it's not belonging to the value) + s.unget(); + + l.FromString(ss); + + return s; + } + +public: + + + /*! + input from standard streams + */ + friend std::istream & operator>>(std::istream & s, UInt & l) + { + return InputFromStream(s, l); + } + + +#ifndef TTMATH_DONT_USE_WCHAR + + /*! + input from standard streams + */ + friend std::wistream & operator>>(std::wistream & s, UInt & l) + { + return InputFromStream(s, l); + } + +#endif + + + /* + Following methods are defined in: + ttmathuint_x86.h + ttmathuint_x86_64.h + ttmathuint_noasm.h + */ + +#ifdef TTMATH_NOASM + static uint AddTwoWords(uint a, uint b, uint carry, uint * result); + static uint SubTwoWords(uint a, uint b, uint carry, uint * result); + +#ifdef TTMATH_PLATFORM64 + + union uint_ + { + struct + { + unsigned int low; // 32 bit + unsigned int high; // 32 bit + } u_; + + uint u; // 64 bit + }; + + + static void DivTwoWords2(uint a,uint b, uint c, uint * r, uint * rest); + static uint DivTwoWordsNormalize(uint_ & a_, uint_ & b_, uint_ & c_); + static uint DivTwoWordsUnnormalize(uint u, uint d); + static unsigned int DivTwoWordsCalculate(uint_ u_, unsigned int u3, uint_ v_); + static void MultiplySubtract(uint_ & u_, unsigned int & u3, unsigned int & q, uint_ v_); + +#endif // TTMATH_PLATFORM64 +#endif // TTMATH_NOASM + + +private: + uint Rcl2_one(uint c); + uint Rcr2_one(uint c); + uint Rcl2(uint bits, uint c); + uint Rcr2(uint bits, uint c); + +public: + static const char * LibTypeStr(); + static LibTypeCode LibType(); + uint Add(const UInt & ss2, uint c=0); + uint AddInt(uint value, uint index = 0); + uint AddTwoInts(uint x2, uint x1, uint index); + static uint AddVector(const uint * ss1, const uint * ss2, uint ss1_size, uint ss2_size, uint * result); + uint Sub(const UInt & ss2, uint c=0); + uint SubInt(uint value, uint index = 0); + static uint SubVector(const uint * ss1, const uint * ss2, uint ss1_size, uint ss2_size, uint * result); + static sint FindLeadingBitInWord(uint x); + static sint FindLowestBitInWord(uint x); + static uint SetBitInWord(uint & value, uint bit); + static void MulTwoWords(uint a, uint b, uint * result_high, uint * result_low); + static void DivTwoWords(uint a,uint b, uint c, uint * r, uint * rest); + +}; + + + +/*! + this specialization is needed in order to not confused the compiler "error: ISO C++ forbids zero-size array" + when compiling Mul3Big2() method +*/ +template<> +class UInt<0> +{ +public: + uint table[1]; + + void Mul2Big(const UInt<0> &, UInt<0> &) { TTMATH_ASSERT(false) }; + void SetZero() { TTMATH_ASSERT(false) }; + uint AddTwoInts(uint, uint, uint) { TTMATH_ASSERT(false) return 0; }; +}; + + +} //namespace + + +#include "ttmathuint_x86.h" +#include "ttmathuint_x86_64.h" +#include "ttmathuint_noasm.h" + +#endif diff --git a/extern/ttmath/ttmathuint_noasm.h b/extern/ttmath/ttmathuint_noasm.h new file mode 100644 index 0000000000..07c73fc499 --- /dev/null +++ b/extern/ttmath/ttmathuint_noasm.h @@ -0,0 +1,1017 @@ +/* + * This file is a part of TTMath Bignum Library + * and is distributed under the (new) BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2006-2010, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name Tomasz Sowa nor the names of contributors to this + * project may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef headerfilettmathuint_noasm +#define headerfilettmathuint_noasm + + +#ifdef TTMATH_NOASM + +/*! + \file ttmathuint_noasm.h + \brief template class UInt with methods without any assembler code + + this file is included at the end of ttmathuint.h +*/ + + +namespace ttmath +{ + + /*! + returning the string represents the currect type of the library + we have following types: + asm_vc_32 - with asm code designed for Microsoft Visual C++ (32 bits) + asm_gcc_32 - with asm code designed for GCC (32 bits) + asm_vc_64 - with asm for VC (64 bit) + asm_gcc_64 - with asm for GCC (64 bit) + no_asm_32 - pure C++ version (32 bit) - without any asm code + no_asm_64 - pure C++ version (64 bit) - without any asm code + */ + template + const char * UInt::LibTypeStr() + { + #ifdef TTMATH_PLATFORM32 + static const char info[] = "no_asm_32"; + #endif + + #ifdef TTMATH_PLATFORM64 + static const char info[] = "no_asm_64"; + #endif + + return info; + } + + + /*! + returning the currect type of the library + */ + template + LibTypeCode UInt::LibType() + { + #ifdef TTMATH_PLATFORM32 + LibTypeCode info = no_asm_32; + #endif + + #ifdef TTMATH_PLATFORM64 + LibTypeCode info = no_asm_64; + #endif + + return info; + } + + + /*! + this method adds two words together + returns carry + + this method is created only when TTMATH_NOASM macro is defined + */ + template + uint UInt::AddTwoWords(uint a, uint b, uint carry, uint * result) + { + uint temp; + + if( carry == 0 ) + { + temp = a + b; + + if( temp < a ) + carry = 1; + } + else + { + carry = 1; + temp = a + b + carry; + + if( temp > a ) // !(temp<=a) + carry = 0; + } + + *result = temp; + + return carry; + } + + + + /*! + this method adding ss2 to the this and adding carry if it's defined + (this = this + ss2 + c) + + c must be zero or one (might be a bigger value than 1) + function returns carry (1) (if it was) + */ + + template + uint UInt::Add(const UInt & ss2, uint c) + { + uint i; + + for(i=0 ; i + uint UInt::AddInt(uint value, uint index) + { + uint i, c; + + TTMATH_ASSERT( index < value_size ) + + + c = AddTwoWords(table[index], value, 0, &table[index]); + + for(i=index+1 ; i + uint UInt::AddTwoInts(uint x2, uint x1, uint index) + { + uint i, c; + + TTMATH_ASSERT( index < value_size - 1 ) + + + c = AddTwoWords(table[index], x1, 0, &table[index]); + c = AddTwoWords(table[index+1], x2, c, &table[index+1]); + + for(i=index+2 ; i + uint UInt::AddVector(const uint * ss1, const uint * ss2, uint ss1_size, uint ss2_size, uint * result) + { + uint i, c = 0; + + TTMATH_ASSERT( ss1_size >= ss2_size ) + + for(i=0 ; i + uint UInt::SubTwoWords(uint a, uint b, uint carry, uint * result) + { + if( carry == 0 ) + { + *result = a - b; + + if( a < b ) + carry = 1; + } + else + { + carry = 1; + *result = a - b - carry; + + if( a > b ) // !(a <= b ) + carry = 0; + } + + return carry; + } + + + + + /*! + this method's subtracting ss2 from the 'this' and subtracting + carry if it has been defined + (this = this - ss2 - c) + + c must be zero or one (might be a bigger value than 1) + function returns carry (1) (if it was) + */ + template + uint UInt::Sub(const UInt & ss2, uint c) + { + uint i; + + for(i=0 ; i + uint UInt::SubInt(uint value, uint index) + { + uint i, c; + + TTMATH_ASSERT( index < value_size ) + + + c = SubTwoWords(table[index], value, 0, &table[index]); + + for(i=index+1 ; i + uint UInt::SubVector(const uint * ss1, const uint * ss2, uint ss1_size, uint ss2_size, uint * result) + { + uint i, c = 0; + + TTMATH_ASSERT( ss1_size >= ss2_size ) + + for(i=0 ; i + uint UInt::Rcl2_one(uint c) + { + uint i, new_c; + + if( c != 0 ) + c = 1; + + for(i=0 ; i this -> return value + + the highest *bit* will be held the 'c' and + the state of one additional bit (on the right hand side) + will be returned + + for example: + let this is 000000010 + after Rcr2_one(1) there'll be 100000001 and Rcr2_one returns 0 + */ + template + uint UInt::Rcr2_one(uint c) + { + sint i; // signed i + uint new_c; + + if( c != 0 ) + c = TTMATH_UINT_HIGHEST_BIT; + + for(i=sint(value_size)-1 ; i>=0 ; --i) + { + new_c = (table[i] & 1) ? TTMATH_UINT_HIGHEST_BIT : 0; + table[i] = (table[i] >> 1) | c; + c = new_c; + } + + c = (c != 0)? 1 : 0; + + TTMATH_LOGC("UInt::Rcr2_one", c) + + return c; + } + + + + + /*! + this method moves all bits into the left hand side + return value <- this <- c + + the lowest *bits* will be held the 'c' and + the state of one additional bit (on the left hand side) + will be returned + + for example: + let this is 001010000 + after Rcl2(3, 1) there'll be 010000111 and Rcl2 returns 1 + */ + template + uint UInt::Rcl2(uint bits, uint c) + { + TTMATH_ASSERT( bits>0 && bits> move; + + for(i=0 ; i> move; + table[i] = (table[i] << bits) | c; + c = new_c; + } + + TTMATH_LOGC("UInt::Rcl2", (c & 1)) + + return (c & 1); + } + + + + + /*! + this method moves all bits into the right hand side + C -> this -> return value + + the highest *bits* will be held the 'c' and + the state of one additional bit (on the right hand side) + will be returned + + for example: + let this is 000000010 + after Rcr2(2, 1) there'll be 110000000 and Rcr2 returns 1 + */ + template + uint UInt::Rcr2(uint bits, uint c) + { + TTMATH_ASSERT( bits>0 && bits=0 ; --i) + { + new_c = table[i] << move; + table[i] = (table[i] >> bits) | c; + c = new_c; + } + + c = (c & TTMATH_UINT_HIGHEST_BIT) ? 1 : 0; + + TTMATH_LOGC("UInt::Rcr2", c) + + return c; + } + + + + + /*! + this method returns the number of the highest set bit in x + if the 'x' is zero this method returns '-1' + */ + template + sint UInt::FindLeadingBitInWord(uint x) + { + if( x == 0 ) + return -1; + + uint bit = TTMATH_BITS_PER_UINT - 1; + + while( (x & TTMATH_UINT_HIGHEST_BIT) == 0 ) + { + x = x << 1; + --bit; + } + + return bit; + } + + + + /*! + this method returns the number of the highest set bit in x + if the 'x' is zero this method returns '-1' + */ + template + sint UInt::FindLowestBitInWord(uint x) + { + if( x == 0 ) + return -1; + + uint bit = 0; + + while( (x & 1) == 0 ) + { + x = x >> 1; + ++bit; + } + + return bit; + } + + + + /*! + this method sets a special bit in the 'value' + and returns the last state of the bit (zero or one) + + bit is from <0,TTMATH_BITS_PER_UINT-1> + + e.g. + uint x = 100; + uint bit = SetBitInWord(x, 3); + now: x = 108 and bit = 0 + */ + template + uint UInt::SetBitInWord(uint & value, uint bit) + { + TTMATH_ASSERT( bit < TTMATH_BITS_PER_UINT ) + + uint mask = 1; + + if( bit > 0 ) + mask = mask << bit; + + uint last = value & mask; + value = value | mask; + + return (last != 0) ? 1 : 0; + } + + + + + + + /*! + * + * Multiplication + * + * + */ + + + /*! + multiplication: result_high:result_low = a * b + result_high - higher word of the result + result_low - lower word of the result + + this methos never returns a carry + this method is used in the second version of the multiplication algorithms + */ + template + void UInt::MulTwoWords(uint a, uint b, uint * result_high, uint * result_low) + { + #ifdef TTMATH_PLATFORM32 + + /* + on 32bit platforms we have defined 'unsigned long long int' type known as 'ulint' in ttmath namespace + this type has 64 bits, then we're using only one multiplication: 32bit * 32bit = 64bit + */ + + union uint_ + { + struct + { + uint low; // 32 bits + uint high; // 32 bits + } u_; + + ulint u; // 64 bits + } res; + + res.u = ulint(a) * ulint(b); // multiply two 32bit words, the result has 64 bits + + *result_high = res.u_.high; + *result_low = res.u_.low; + + #else + + /* + 64 bits platforms + + we don't have a native type which has 128 bits + then we're splitting 'a' and 'b' to 4 parts (high and low halves) + and using 4 multiplications (with additions and carry correctness) + */ + + uint_ a_; + uint_ b_; + uint_ res_high1, res_high2; + uint_ res_low1, res_low2; + + a_.u = a; + b_.u = b; + + /* + the multiplication is as follows (schoolbook algorithm with O(n^2) ): + + 32 bits 32 bits + + +--------------------------------+ + | a_.u_.high | a_.u_.low | + +--------------------------------+ + | b_.u_.high | b_.u_.low | + +--------------------------------+--------------------------------+ + | res_high1.u | res_low1.u | + +--------------------------------+--------------------------------+ + | res_high2.u | res_low2.u | + +--------------------------------+--------------------------------+ + + 64 bits 64 bits + */ + + + uint_ temp; + + res_low1.u = uint(b_.u_.low) * uint(a_.u_.low); + + temp.u = uint(res_low1.u_.high) + uint(b_.u_.low) * uint(a_.u_.high); + res_low1.u_.high = temp.u_.low; + res_high1.u_.low = temp.u_.high; + res_high1.u_.high = 0; + + res_low2.u_.low = 0; + temp.u = uint(b_.u_.high) * uint(a_.u_.low); + res_low2.u_.high = temp.u_.low; + + res_high2.u = uint(b_.u_.high) * uint(a_.u_.high) + uint(temp.u_.high); + + uint c = AddTwoWords(res_low1.u, res_low2.u, 0, &res_low2.u); + AddTwoWords(res_high1.u, res_high2.u, c, &res_high2.u); // there is no carry from here + + *result_high = res_high2.u; + *result_low = res_low2.u; + + #endif + } + + + + + /*! + * + * Division + * + * + */ + + + /*! + this method calculates 64bits word a:b / 32bits c (a higher, b lower word) + r = a:b / c and rest - remainder + + * + * WARNING: + * the c has to be suitably large for the result being keeped in one word, + * if c is equal zero there'll be a hardware interruption (0) + * and probably the end of your program + * + */ + template + void UInt::DivTwoWords(uint a, uint b, uint c, uint * r, uint * rest) + { + // (a < c ) for the result to be one word + TTMATH_ASSERT( c != 0 && a < c ) + + #ifdef TTMATH_PLATFORM32 + + union + { + struct + { + uint low; // 32 bits + uint high; // 32 bits + } u_; + + ulint u; // 64 bits + } ab; + + ab.u_.high = a; + ab.u_.low = b; + + *r = uint(ab.u / c); + *rest = uint(ab.u % c); + + #else + + uint_ c_; + c_.u = c; + + + if( a == 0 ) + { + *r = b / c; + *rest = b % c; + } + else + if( c_.u_.high == 0 ) + { + // higher half of 'c' is zero + // then higher half of 'a' is zero too (look at the asserts at the beginning - 'a' is smaller than 'c') + uint_ a_, b_, res_, temp1, temp2; + + a_.u = a; + b_.u = b; + + temp1.u_.high = a_.u_.low; + temp1.u_.low = b_.u_.high; + + res_.u_.high = (unsigned int)(temp1.u / c); + temp2.u_.high = (unsigned int)(temp1.u % c); + temp2.u_.low = b_.u_.low; + + res_.u_.low = (unsigned int)(temp2.u / c); + *rest = temp2.u % c; + + *r = res_.u; + } + else + { + return DivTwoWords2(a, b, c, r, rest); + } + + #endif + } + + +#ifdef TTMATH_PLATFORM64 + + + /*! + this method is available only on 64bit platforms + + the same algorithm like the third division algorithm in ttmathuint.h + but now with the radix=2^32 + */ + template + void UInt::DivTwoWords2(uint a, uint b, uint c, uint * r, uint * rest) + { + // a is not zero + // c_.u_.high is not zero + + uint_ a_, b_, c_, u_, q_; + unsigned int u3; // 32 bit + + a_.u = a; + b_.u = b; + c_.u = c; + + // normalizing + uint d = DivTwoWordsNormalize(a_, b_, c_); + + // loop from j=1 to j=0 + // the first step (for j=2) is skipped because our result is only in one word, + // (first 'q' were 0 and nothing would be changed) + u_.u_.high = a_.u_.high; + u_.u_.low = a_.u_.low; + u3 = b_.u_.high; + q_.u_.high = DivTwoWordsCalculate(u_, u3, c_); + MultiplySubtract(u_, u3, q_.u_.high, c_); + + u_.u_.high = u_.u_.low; + u_.u_.low = u3; + u3 = b_.u_.low; + q_.u_.low = DivTwoWordsCalculate(u_, u3, c_); + MultiplySubtract(u_, u3, q_.u_.low, c_); + + *r = q_.u; + + // unnormalizing for the remainder + u_.u_.high = u_.u_.low; + u_.u_.low = u3; + *rest = DivTwoWordsUnnormalize(u_.u, d); + } + + + + + template + uint UInt::DivTwoWordsNormalize(uint_ & a_, uint_ & b_, uint_ & c_) + { + uint d = 0; + + for( ; (c_.u & TTMATH_UINT_HIGHEST_BIT) == 0 ; ++d ) + { + c_.u = c_.u << 1; + + uint bc = b_.u & TTMATH_UINT_HIGHEST_BIT; // carry from 'b' + + b_.u = b_.u << 1; + a_.u = a_.u << 1; // carry bits from 'a' are simply skipped + + if( bc ) + a_.u = a_.u | 1; + } + + return d; + } + + + template + uint UInt::DivTwoWordsUnnormalize(uint u, uint d) + { + if( d == 0 ) + return u; + + u = u >> d; + + return u; + } + + + template + unsigned int UInt::DivTwoWordsCalculate(uint_ u_, unsigned int u3, uint_ v_) + { + bool next_test; + uint_ qp_, rp_, temp_; + + qp_.u = u_.u / uint(v_.u_.high); + rp_.u = u_.u % uint(v_.u_.high); + + TTMATH_ASSERT( qp_.u_.high==0 || qp_.u_.high==1 ) + + do + { + bool decrease = false; + + if( qp_.u_.high == 1 ) + decrease = true; + else + { + temp_.u_.high = rp_.u_.low; + temp_.u_.low = u3; + + if( qp_.u * uint(v_.u_.low) > temp_.u ) + decrease = true; + } + + next_test = false; + + if( decrease ) + { + --qp_.u; + rp_.u += v_.u_.high; + + if( rp_.u_.high == 0 ) + next_test = true; + } + } + while( next_test ); + + return qp_.u_.low; + } + + + template + void UInt::MultiplySubtract(uint_ & u_, unsigned int & u3, unsigned int & q, uint_ v_) + { + uint_ temp_; + + uint res_high; + uint res_low; + + MulTwoWords(v_.u, q, &res_high, &res_low); + + uint_ sub_res_high_; + uint_ sub_res_low_; + + temp_.u_.high = u_.u_.low; + temp_.u_.low = u3; + + uint c = SubTwoWords(temp_.u, res_low, 0, &sub_res_low_.u); + + temp_.u_.high = 0; + temp_.u_.low = u_.u_.high; + c = SubTwoWords(temp_.u, res_high, c, &sub_res_high_.u); + + if( c ) + { + --q; + + c = AddTwoWords(sub_res_low_.u, v_.u, 0, &sub_res_low_.u); + AddTwoWords(sub_res_high_.u, 0, c, &sub_res_high_.u); + } + + u_.u_.high = sub_res_high_.u_.low; + u_.u_.low = sub_res_low_.u_.high; + u3 = sub_res_low_.u_.low; + } + +#endif // #ifdef TTMATH_PLATFORM64 + + + +} //namespace + + +#endif //ifdef TTMATH_NOASM +#endif + + + + diff --git a/extern/ttmath/ttmathuint_x86.h b/extern/ttmath/ttmathuint_x86.h new file mode 100644 index 0000000000..1dd087f524 --- /dev/null +++ b/extern/ttmath/ttmathuint_x86.h @@ -0,0 +1,1602 @@ +/* + * This file is a part of TTMath Bignum Library + * and is distributed under the (new) BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2006-2009, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name Tomasz Sowa nor the names of contributors to this + * project may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + + + +#ifndef headerfilettmathuint_x86 +#define headerfilettmathuint_x86 + + +#ifndef TTMATH_NOASM +#ifdef TTMATH_PLATFORM32 + + +/*! + \file ttmathuint_x86.h + \brief template class UInt with assembler code for 32bit x86 processors + + this file is included at the end of ttmathuint.h +*/ + + + +/*! + \brief a namespace for the TTMath library +*/ +namespace ttmath +{ + + /*! + returning the string represents the currect type of the library + we have following types: + asm_vc_32 - with asm code designed for Microsoft Visual C++ (32 bits) + asm_gcc_32 - with asm code designed for GCC (32 bits) + asm_vc_64 - with asm for VC (64 bit) + asm_gcc_64 - with asm for GCC (64 bit) + no_asm_32 - pure C++ version (32 bit) - without any asm code + no_asm_64 - pure C++ version (64 bit) - without any asm code + */ + template + const char * UInt::LibTypeStr() + { + #ifndef __GNUC__ + static const char info[] = "asm_vc_32"; + #endif + + #ifdef __GNUC__ + static const char info[] = "asm_gcc_32"; + #endif + + return info; + } + + + /*! + returning the currect type of the library + */ + template + LibTypeCode UInt::LibType() + { + #ifndef __GNUC__ + LibTypeCode info = asm_vc_32; + #endif + + #ifdef __GNUC__ + LibTypeCode info = asm_gcc_32; + #endif + + return info; + } + + + + /*! + * + * basic mathematic functions + * + */ + + + /*! + adding ss2 to the this and adding carry if it's defined + (this = this + ss2 + c) + + c must be zero or one (might be a bigger value than 1) + function returns carry (1) (if it has been) + */ + template + uint UInt::Add(const UInt & ss2, uint c) + { + uint b = value_size; + uint * p1 = table; + uint * p2 = const_cast(ss2.table); + + // we don't have to use TTMATH_REFERENCE_ASSERT here + // this algorithm doesn't require it + + #ifndef __GNUC__ + + // this part might be compiled with for example visual c + + __asm + { + push eax + push ebx + push ecx + push edx + push esi + + mov ecx,[b] + + mov ebx,[p1] + mov esi,[p2] + + xor edx,edx // edx=0 + mov eax,[c] + neg eax // CF=1 if rax!=0 , CF=0 if rax==0 + + ttmath_loop: + mov eax,[esi+edx*4] + adc [ebx+edx*4],eax + + inc edx + dec ecx + jnz ttmath_loop + + adc ecx, ecx + mov [c], ecx + + pop esi + pop edx + pop ecx + pop ebx + pop eax + } + + + + #endif + + + #ifdef __GNUC__ + uint dummy, dummy2; + // this part should be compiled with gcc + + __asm__ __volatile__( + + "xorl %%edx, %%edx \n" + "negl %%eax \n" // CF=1 if rax!=0 , CF=0 if rax==0 + + "1: \n" + "movl (%%esi,%%edx,4), %%eax \n" + "adcl %%eax, (%%ebx,%%edx,4) \n" + + "incl %%edx \n" + "decl %%ecx \n" + "jnz 1b \n" + + "adc %%ecx, %%ecx \n" + + : "=c" (c), "=a" (dummy), "=d" (dummy2) + : "0" (b), "1" (c), "b" (p1), "S" (p2) + : "cc", "memory" ); + #endif + + TTMATH_LOGC("UInt::Add", c) + + return c; + } + + + + /*! + adding one word (at a specific position) + and returning a carry (if it has been) + + e.g. + + if we've got (value_size=3): + table[0] = 10; + table[1] = 30; + table[2] = 5; + and we call: + AddInt(2,1) + then it'll be: + table[0] = 10; + table[1] = 30 + 2; + table[2] = 5; + + of course if there was a carry from table[2] it would be returned + */ + template + uint UInt::AddInt(uint value, uint index) + { + uint b = value_size; + uint * p1 = table; + uint c; + + TTMATH_ASSERT( index < value_size ) + + #ifndef __GNUC__ + + __asm + { + push eax + push ebx + push ecx + push edx + + mov ecx, [b] + sub ecx, [index] + + mov edx, [index] + mov ebx, [p1] + + mov eax, [value] + + ttmath_loop: + add [ebx+edx*4], eax + jnc ttmath_end + + mov eax, 1 + inc edx + dec ecx + jnz ttmath_loop + + ttmath_end: + setc al + movzx edx, al + mov [c], edx + + pop edx + pop ecx + pop ebx + pop eax + } + + #endif + + + #ifdef __GNUC__ + uint dummy, dummy2; + + __asm__ __volatile__( + + "subl %%edx, %%ecx \n" + + "1: \n" + "addl %%eax, (%%ebx,%%edx,4) \n" + "jnc 2f \n" + + "movl $1, %%eax \n" + "incl %%edx \n" + "decl %%ecx \n" + "jnz 1b \n" + + "2: \n" + "setc %%al \n" + "movzx %%al, %%edx \n" + + : "=d" (c), "=a" (dummy), "=c" (dummy2) + : "0" (index), "1" (value), "2" (b), "b" (p1) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::AddInt", c) + + return c; + } + + + + + /*! + adding only two unsigned words to the existing value + and these words begin on the 'index' position + (it's used in the multiplication algorithm 2) + + index should be equal or smaller than value_size-2 (index <= value_size-2) + x1 - lower word, x2 - higher word + + for example if we've got value_size equal 4 and: + table[0] = 3 + table[1] = 4 + table[2] = 5 + table[3] = 6 + then let + x1 = 10 + x2 = 20 + and + index = 1 + + the result of this method will be: + table[0] = 3 + table[1] = 4 + x1 = 14 + table[2] = 5 + x2 = 25 + table[3] = 6 + + and no carry at the end of table[3] + + (of course if there was a carry in table[2](5+20) then + this carry would be passed to the table[3] etc.) + */ + template + uint UInt::AddTwoInts(uint x2, uint x1, uint index) + { + uint b = value_size; + uint * p1 = table; + uint c; + + TTMATH_ASSERT( index < value_size - 1 ) + + #ifndef __GNUC__ + __asm + { + push eax + push ebx + push ecx + push edx + + mov ecx, [b] + sub ecx, [index] + + mov ebx, [p1] + mov edx, [index] + + mov eax, [x1] + add [ebx+edx*4], eax + inc edx + dec ecx + + mov eax, [x2] + + ttmath_loop: + adc [ebx+edx*4], eax + jnc ttmath_end + + mov eax, 0 + inc edx + dec ecx + jnz ttmath_loop + + ttmath_end: + setc al + movzx edx, al + mov [c], edx + + pop edx + pop ecx + pop ebx + pop eax + + } + #endif + + + #ifdef __GNUC__ + uint dummy, dummy2; + + __asm__ __volatile__( + + "subl %%edx, %%ecx \n" + + "addl %%esi, (%%ebx,%%edx,4) \n" + "incl %%edx \n" + "decl %%ecx \n" + + "1: \n" + "adcl %%eax, (%%ebx,%%edx,4) \n" + "jnc 2f \n" + + "mov $0, %%eax \n" + "incl %%edx \n" + "decl %%ecx \n" + "jnz 1b \n" + + "2: \n" + "setc %%al \n" + "movzx %%al, %%eax \n" + + : "=a" (c), "=c" (dummy), "=d" (dummy2) + : "0" (x2), "1" (b), "2" (index), "b" (p1), "S" (x1) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::AddTwoInts", c) + + return c; + } + + + + /*! + this static method addes one vector to the other + 'ss1' is larger in size or equal to 'ss2' + + ss1 points to the first (larger) vector + ss2 points to the second vector + ss1_size - size of the ss1 (and size of the result too) + ss2_size - size of the ss2 + result - is the result vector (which has size the same as ss1: ss1_size) + + Example: ss1_size is 5, ss2_size is 3 + ss1: ss2: result (output): + 5 1 5+1 + 4 3 4+3 + 2 7 2+7 + 6 6 + 9 9 + of course the carry is propagated and will be returned from the last item + (this method is used by the Karatsuba multiplication algorithm) + */ + template + uint UInt::AddVector(const uint * ss1, const uint * ss2, uint ss1_size, uint ss2_size, uint * result) + { + TTMATH_ASSERT( ss1_size >= ss2_size ) + + uint rest = ss1_size - ss2_size; + uint c; + + #ifndef __GNUC__ + + // this part might be compiled with for example visual c + __asm + { + pushad + + mov ecx, [ss2_size] + xor edx, edx // edx = 0, cf = 0 + + mov esi, [ss1] + mov ebx, [ss2] + mov edi, [result] + + ttmath_loop: + mov eax, [esi+edx*4] + adc eax, [ebx+edx*4] + mov [edi+edx*4], eax + + inc edx + dec ecx + jnz ttmath_loop + + adc ecx, ecx // ecx has the cf state + + mov ebx, [rest] + or ebx, ebx + jz ttmath_end + + xor ebx, ebx // ebx = 0 + neg ecx // setting cf from ecx + mov ecx, [rest] // ecx is != 0 + + ttmath_loop2: + mov eax, [esi+edx*4] + adc eax, ebx + mov [edi+edx*4], eax + + inc edx + dec ecx + jnz ttmath_loop2 + + adc ecx, ecx + + ttmath_end: + mov [c], ecx + + popad + } + + #endif + + + #ifdef __GNUC__ + + // this part should be compiled with gcc + uint dummy1, dummy2, dummy3; + + __asm__ __volatile__( + "push %%edx \n" + "xor %%edx, %%edx \n" // edx = 0, cf = 0 + "1: \n" + "mov (%%esi,%%edx,4), %%eax \n" + "adc (%%ebx,%%edx,4), %%eax \n" + "mov %%eax, (%%edi,%%edx,4) \n" + + "inc %%edx \n" + "dec %%ecx \n" + "jnz 1b \n" + + "adc %%ecx, %%ecx \n" // ecx has the cf state + "pop %%eax \n" // eax = rest + + "or %%eax, %%eax \n" + "jz 3f \n" + + "xor %%ebx, %%ebx \n" // ebx = 0 + "neg %%ecx \n" // setting cf from ecx + "mov %%eax, %%ecx \n" // ecx=rest and is != 0 + "2: \n" + "mov (%%esi, %%edx, 4), %%eax \n" + "adc %%ebx, %%eax \n" + "mov %%eax, (%%edi, %%edx, 4) \n" + + "inc %%edx \n" + "dec %%ecx \n" + "jnz 2b \n" + + "adc %%ecx, %%ecx \n" + "3: \n" + + : "=a" (dummy1), "=b" (dummy2), "=c" (c), "=d" (dummy3) + : "1" (ss2), "2" (ss2_size), "3" (rest), "S" (ss1), "D" (result) + : "cc", "memory" ); + + #endif + + TTMATH_VECTOR_LOGC("UInt::AddVector", c, result, ss1_size) + + return c; + } + + + /*! + subtracting ss2 from the 'this' and subtracting + carry if it has been defined + (this = this - ss2 - c) + + c must be zero or one (might be a bigger value than 1) + function returns carry (1) (if it has been) + */ + template + uint UInt::Sub(const UInt & ss2, uint c) + { + uint b = value_size; + uint * p1 = table; + uint * p2 = const_cast(ss2.table); + + // we don't have to use TTMATH_REFERENCE_ASSERT here + // this algorithm doesn't require it + + #ifndef __GNUC__ + + __asm + { + push eax + push ebx + push ecx + push edx + push esi + + mov ecx,[b] + + mov ebx,[p1] + mov esi,[p2] + + xor edx,edx // edx=0 + mov eax,[c] + neg eax // CF=1 if rax!=0 , CF=0 if rax==0 + + ttmath_loop: + mov eax,[esi+edx*4] + sbb [ebx+edx*4],eax + + inc edx + dec ecx + jnz ttmath_loop + + adc ecx, ecx + mov [c], ecx + + pop esi + pop edx + pop ecx + pop ebx + pop eax + } + + #endif + + + #ifdef __GNUC__ + uint dummy, dummy2; + + __asm__ __volatile__( + + "xorl %%edx, %%edx \n" + "negl %%eax \n" // CF=1 if rax!=0 , CF=0 if rax==0 + + "1: \n" + "movl (%%esi,%%edx,4), %%eax \n" + "sbbl %%eax, (%%ebx,%%edx,4) \n" + + "incl %%edx \n" + "decl %%ecx \n" + "jnz 1b \n" + + "adc %%ecx, %%ecx \n" + + : "=c" (c), "=a" (dummy), "=d" (dummy2) + : "0" (b), "1" (c), "b" (p1), "S" (p2) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::Sub", c) + + return c; + } + + + + + /*! + this method subtracts one word (at a specific position) + and returns a carry (if it was) + + e.g. + + if we've got (value_size=3): + table[0] = 10; + table[1] = 30; + table[2] = 5; + and we call: + SubInt(2,1) + then it'll be: + table[0] = 10; + table[1] = 30 - 2; + table[2] = 5; + + of course if there was a carry from table[2] it would be returned + */ + template + uint UInt::SubInt(uint value, uint index) + { + uint b = value_size; + uint * p1 = table; + uint c; + + TTMATH_ASSERT( index < value_size ) + + #ifndef __GNUC__ + + __asm + { + push eax + push ebx + push ecx + push edx + + mov ecx, [b] + sub ecx, [index] + + mov edx, [index] + mov ebx, [p1] + + mov eax, [value] + + ttmath_loop: + sub [ebx+edx*4], eax + jnc ttmath_end + + mov eax, 1 + inc edx + dec ecx + jnz ttmath_loop + + ttmath_end: + setc al + movzx edx, al + mov [c], edx + + pop edx + pop ecx + pop ebx + pop eax + } + + #endif + + + #ifdef __GNUC__ + uint dummy, dummy2; + + __asm__ __volatile__( + + "subl %%edx, %%ecx \n" + + "1: \n" + "subl %%eax, (%%ebx,%%edx,4) \n" + "jnc 2f \n" + + "movl $1, %%eax \n" + "incl %%edx \n" + "decl %%ecx \n" + "jnz 1b \n" + + "2: \n" + "setc %%al \n" + "movzx %%al, %%edx \n" + + : "=d" (c), "=a" (dummy), "=c" (dummy2) + : "0" (index), "1" (value), "2" (b), "b" (p1) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::SubInt", c) + + return c; + } + + + + /*! + this static method subtractes one vector from the other + 'ss1' is larger in size or equal to 'ss2' + + ss1 points to the first (larger) vector + ss2 points to the second vector + ss1_size - size of the ss1 (and size of the result too) + ss2_size - size of the ss2 + result - is the result vector (which has size the same as ss1: ss1_size) + + Example: ss1_size is 5, ss2_size is 3 + ss1: ss2: result (output): + 5 1 5-1 + 4 3 4-3 + 2 7 2-7 + 6 6-1 (the borrow from previous item) + 9 9 + return (carry): 0 + of course the carry (borrow) is propagated and will be returned from the last item + (this method is used by the Karatsuba multiplication algorithm) + */ + template + uint UInt::SubVector(const uint * ss1, const uint * ss2, uint ss1_size, uint ss2_size, uint * result) + { + TTMATH_ASSERT( ss1_size >= ss2_size ) + + uint rest = ss1_size - ss2_size; + uint c; + + #ifndef __GNUC__ + + // this part might be compiled with for example visual c + + /* + the asm code is nearly the same as in AddVector + only two instructions 'adc' are changed to 'sbb' + */ + __asm + { + pushad + + mov ecx, [ss2_size] + xor edx, edx // edx = 0, cf = 0 + + mov esi, [ss1] + mov ebx, [ss2] + mov edi, [result] + + ttmath_loop: + mov eax, [esi+edx*4] + sbb eax, [ebx+edx*4] + mov [edi+edx*4], eax + + inc edx + dec ecx + jnz ttmath_loop + + adc ecx, ecx // ecx has the cf state + + mov ebx, [rest] + or ebx, ebx + jz ttmath_end + + xor ebx, ebx // ebx = 0 + neg ecx // setting cf from ecx + mov ecx, [rest] // ecx is != 0 + + ttmath_loop2: + mov eax, [esi+edx*4] + sbb eax, ebx + mov [edi+edx*4], eax + + inc edx + dec ecx + jnz ttmath_loop2 + + adc ecx, ecx + + ttmath_end: + mov [c], ecx + + popad + } + + #endif + + + #ifdef __GNUC__ + + // this part should be compiled with gcc + uint dummy1, dummy2, dummy3; + + __asm__ __volatile__( + "push %%edx \n" + "xor %%edx, %%edx \n" // edx = 0, cf = 0 + "1: \n" + "mov (%%esi,%%edx,4), %%eax \n" + "sbb (%%ebx,%%edx,4), %%eax \n" + "mov %%eax, (%%edi,%%edx,4) \n" + + "inc %%edx \n" + "dec %%ecx \n" + "jnz 1b \n" + + "adc %%ecx, %%ecx \n" // ecx has the cf state + "pop %%eax \n" // eax = rest + + "or %%eax, %%eax \n" + "jz 3f \n" + + "xor %%ebx, %%ebx \n" // ebx = 0 + "neg %%ecx \n" // setting cf from ecx + "mov %%eax, %%ecx \n" // ecx=rest and is != 0 + "2: \n" + "mov (%%esi, %%edx, 4), %%eax \n" + "sbb %%ebx, %%eax \n" + "mov %%eax, (%%edi, %%edx, 4) \n" + + "inc %%edx \n" + "dec %%ecx \n" + "jnz 2b \n" + + "adc %%ecx, %%ecx \n" + "3: \n" + + : "=a" (dummy1), "=b" (dummy2), "=c" (c), "=d" (dummy3) + : "1" (ss2), "2" (ss2_size), "3" (rest), "S" (ss1), "D" (result) + : "cc", "memory" ); + + #endif + + TTMATH_VECTOR_LOGC("UInt::SubVector", c, result, ss1_size) + + return c; + } + + + + /*! + this method moves all bits into the left hand side + return value <- this <- c + + the lowest *bit* will be held the 'c' and + the state of one additional bit (on the left hand side) + will be returned + + for example: + let this is 001010000 + after Rcl2_one(1) there'll be 010100001 and Rcl2_one returns 0 + */ + template + uint UInt::Rcl2_one(uint c) + { + uint b = value_size; + uint * p1 = table; + + #ifndef __GNUC__ + __asm + { + push ebx + push ecx + push edx + + mov ebx, [p1] + xor edx, edx + mov ecx, [c] + neg ecx + mov ecx, [b] + + ttmath_loop: + rcl dword ptr [ebx+edx*4], 1 + + inc edx + dec ecx + jnz ttmath_loop + + adc ecx, ecx + mov [c], ecx + + pop edx + pop ecx + pop ebx + } + #endif + + + #ifdef __GNUC__ + uint dummy, dummy2; + + __asm__ __volatile__( + + "xorl %%edx, %%edx \n" // edx=0 + "negl %%eax \n" // CF=1 if eax!=0 , CF=0 if eax==0 + + "1: \n" + "rcll $1, (%%ebx, %%edx, 4) \n" + + "incl %%edx \n" + "decl %%ecx \n" + "jnz 1b \n" + + "adcl %%ecx, %%ecx \n" + + : "=c" (c), "=a" (dummy), "=d" (dummy2) + : "0" (b), "1" (c), "b" (p1) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::Rcl2_one", c) + + return c; + } + + + + /*! + this method moves all bits into the right hand side + c -> this -> return value + + the highest *bit* will be held the 'c' and + the state of one additional bit (on the right hand side) + will be returned + + for example: + let this is 000000010 + after Rcr2_one(1) there'll be 100000001 and Rcr2_one returns 0 + */ + template + uint UInt::Rcr2_one(uint c) + { + uint b = value_size; + uint * p1 = table; + + #ifndef __GNUC__ + __asm + { + push ebx + push ecx + + mov ebx, [p1] + mov ecx, [c] + neg ecx + mov ecx, [b] + + ttmath_loop: + rcr dword ptr [ebx+ecx*4-4], 1 + + dec ecx + jnz ttmath_loop + + adc ecx, ecx + mov [c], ecx + + pop ecx + pop ebx + } + #endif + + + #ifdef __GNUC__ + uint dummy; + + __asm__ __volatile__( + + "negl %%eax \n" // CF=1 if eax!=0 , CF=0 if eax==0 + + "1: \n" + "rcrl $1, -4(%%ebx, %%ecx, 4) \n" + + "decl %%ecx \n" + "jnz 1b \n" + + "adcl %%ecx, %%ecx \n" + + : "=c" (c), "=a" (dummy) + : "0" (b), "1" (c), "b" (p1) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::Rcr2_one", c) + + return c; + } + + + +#ifdef _MSC_VER +#pragma warning (disable : 4731) +//warning C4731: frame pointer register 'ebp' modified by inline assembly code +#endif + + + + /*! + this method moves all bits into the left hand side + return value <- this <- c + + the lowest *bits* will be held the 'c' and + the state of one additional bit (on the left hand side) + will be returned + + for example: + let this is 001010000 + after Rcl2(3, 1) there'll be 010000111 and Rcl2 returns 1 + */ + template + uint UInt::Rcl2(uint bits, uint c) + { + TTMATH_ASSERT( bits>0 && bits edx -> cf) (cl times) + "movl %%edx, %%ebp \n" // ebp = edx = mask + "movl %%esi, %%ecx \n" + + "xorl %%edx, %%edx \n" + "movl %%edx, %%esi \n" + "orl %%eax, %%eax \n" + "cmovnz %%ebp, %%esi \n" // if(c) esi=mask else esi=0 + + "1: \n" + "roll %%cl, (%%ebx,%%edx,4) \n" + + "movl (%%ebx,%%edx,4), %%eax \n" + "andl %%ebp, %%eax \n" + "xorl %%eax, (%%ebx,%%edx,4) \n" + "orl %%esi, (%%ebx,%%edx,4) \n" + "movl %%eax, %%esi \n" + + "incl %%edx \n" + "decl %%edi \n" + "jnz 1b \n" + + "and $1, %%eax \n" + + "pop %%ebp \n" + + : "=a" (c), "=D" (dummy), "=S" (dummy2), "=d" (dummy3) + : "0" (c), "1" (b), "b" (p1), "c" (bits) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::Rcl2", c) + + return c; + } + + + + + /*! + this method moves all bits into the right hand side + C -> this -> return value + + the highest *bits* will be held the 'c' and + the state of one additional bit (on the right hand side) + will be returned + + for example: + let this is 000000010 + after Rcr2(2, 1) there'll be 110000000 and Rcr2 returns 1 + */ + template + uint UInt::Rcr2(uint bits, uint c) + { + TTMATH_ASSERT( bits>0 && bits + sint UInt::FindLeadingBitInWord(uint x) + { + sint result; + + #ifndef __GNUC__ + __asm + { + push eax + push edx + + mov edx,-1 + bsr eax,[x] + cmovz eax,edx + mov [result], eax + + pop edx + pop eax + } + #endif + + + #ifdef __GNUC__ + uint dummy; + + __asm__ ( + + "movl $-1, %1 \n" + "bsrl %2, %0 \n" + "cmovz %1, %0 \n" + + : "=r" (result), "=&r" (dummy) + : "r" (x) + : "cc" ); + + #endif + + return result; + } + + + + /* + this method returns the number of the smallest set bit in one 32-bit word + if the 'x' is zero this method returns '-1' + */ + template + sint UInt::FindLowestBitInWord(uint x) + { + sint result; + + #ifndef __GNUC__ + __asm + { + push eax + push edx + + mov edx,-1 + bsf eax,[x] + cmovz eax,edx + mov [result], eax + + pop edx + pop eax + } + #endif + + + #ifdef __GNUC__ + uint dummy; + + __asm__ ( + + "movl $-1, %1 \n" + "bsfl %2, %0 \n" + "cmovz %1, %0 \n" + + : "=r" (result), "=&r" (dummy) + : "r" (x) + : "cc" ); + + #endif + + return result; + } + + + + /*! + this method sets a special bit in the 'value' + and returns the last state of the bit (zero or one) + + bit is from <0,31> + e.g. + uint x = 100; + uint bit = SetBitInWord(x, 3); + now: x = 108 and bit = 0 + */ + template + uint UInt::SetBitInWord(uint & value, uint bit) + { + TTMATH_ASSERT( bit < TTMATH_BITS_PER_UINT ) + + uint old_bit; + uint v = value; + + #ifndef __GNUC__ + __asm + { + push ebx + push eax + + mov eax, [v] + mov ebx, [bit] + bts eax, ebx + mov [v], eax + + setc bl + movzx ebx, bl + mov [old_bit], ebx + + pop eax + pop ebx + } + #endif + + + #ifdef __GNUC__ + __asm__ ( + + "btsl %%ebx, %%eax \n" + "setc %%bl \n" + "movzx %%bl, %%ebx \n" + + : "=a" (v), "=b" (old_bit) + : "0" (v), "1" (bit) + : "cc" ); + + #endif + + value = v; + + return old_bit; + } + + + + + /*! + multiplication: result_high:result_low = a * b + result_high - higher word of the result + result_low - lower word of the result + + this methos never returns a carry + this method is used in the second version of the multiplication algorithms + */ + template + void UInt::MulTwoWords(uint a, uint b, uint * result_high, uint * result_low) + { + /* + we must use these temporary variables in order to inform the compilator + that value pointed with result1 and result2 has changed + + this has no effect in visual studio but it's useful when + using gcc and options like -Ox + */ + uint result1_; + uint result2_; + + #ifndef __GNUC__ + + __asm + { + push eax + push edx + + mov eax, [a] + mul dword ptr [b] + + mov [result2_], edx + mov [result1_], eax + + pop edx + pop eax + } + + #endif + + + #ifdef __GNUC__ + + __asm__ ( + + "mull %%edx \n" + + : "=a" (result1_), "=d" (result2_) + : "0" (a), "1" (b) + : "cc" ); + + #endif + + + *result_low = result1_; + *result_high = result2_; + } + + + + + + /*! + * + * Division + * + * + */ + + + + + /*! + this method calculates 64bits word a:b / 32bits c (a higher, b lower word) + r = a:b / c and rest - remainder + + * + * WARNING: + * if r (one word) is too small for the result or c is equal zero + * there'll be a hardware interruption (0) + * and probably the end of your program + * + */ + template + void UInt::DivTwoWords(uint a, uint b, uint c, uint * r, uint * rest) + { + uint r_; + uint rest_; + /* + these variables have similar meaning like those in + the multiplication algorithm MulTwoWords + */ + + TTMATH_ASSERT( c != 0 ) + + #ifndef __GNUC__ + __asm + { + push eax + push edx + + mov edx, [a] + mov eax, [b] + div dword ptr [c] + + mov [r_], eax + mov [rest_], edx + + pop edx + pop eax + } + #endif + + + #ifdef __GNUC__ + + __asm__ ( + + "divl %%ecx \n" + + : "=a" (r_), "=d" (rest_) + : "0" (b), "1" (a), "c" (c) + : "cc" ); + + #endif + + + *r = r_; + *rest = rest_; + + } + + + +} //namespace + + + +#endif //ifdef TTMATH_PLATFORM32 +#endif //ifndef TTMATH_NOASM +#endif diff --git a/extern/ttmath/ttmathuint_x86_64.h b/extern/ttmath/ttmathuint_x86_64.h new file mode 100644 index 0000000000..188fc5e7bd --- /dev/null +++ b/extern/ttmath/ttmathuint_x86_64.h @@ -0,0 +1,1146 @@ +/* + * This file is a part of TTMath Bignum Library + * and is distributed under the (new) BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2006-2010, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name Tomasz Sowa nor the names of contributors to this + * project may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef headerfilettmathuint_x86_64 +#define headerfilettmathuint_x86_64 + + +#ifndef TTMATH_NOASM +#ifdef TTMATH_PLATFORM64 + + +/*! + \file ttmathuint_x86_64.h + \brief template class UInt with assembler code for 64bit x86_64 processors + + this file is included at the end of ttmathuint.h +*/ + +#ifndef __GNUC__ +#include +#endif + + +namespace ttmath +{ + + #ifndef __GNUC__ + + extern "C" + { + uint __fastcall ttmath_adc_x64(uint* p1, const uint* p2, uint nSize, uint c); + uint __fastcall ttmath_addindexed_x64(uint* p1, uint nSize, uint nPos, uint nValue); + uint __fastcall ttmath_addindexed2_x64(uint* p1, uint nSize, uint nPos, uint nValue1, uint nValue2); + uint __fastcall ttmath_addvector_x64(const uint * ss1, const uint * ss2, uint ss1_size, uint ss2_size, uint * result); + uint __fastcall ttmath_sbb_x64(uint* p1, const uint* p2, uint nSize, uint c); + uint __fastcall ttmath_subindexed_x64(uint* p1, uint nSize, uint nPos, uint nValue); + uint __fastcall ttmath_subvector_x64(const uint * ss1, const uint * ss2, uint ss1_size, uint ss2_size, uint * result); + uint __fastcall ttmath_rcl_x64(uint* p1, uint nSize, uint nLowestBit); + uint __fastcall ttmath_rcr_x64(uint* p1, uint nSize, uint nLowestBit); + uint __fastcall ttmath_div_x64(uint* pnValHi, uint* pnValLo, uint nDiv); + uint __fastcall ttmath_rcl2_x64(uint* p1, uint nSize, uint nBits, uint c); + uint __fastcall ttmath_rcr2_x64(uint* p1, uint nSize, uint nBits, uint c); + }; + #endif + + + /*! + returning the string represents the currect type of the library + we have following types: + asm_vc_32 - with asm code designed for Microsoft Visual C++ (32 bits) + asm_gcc_32 - with asm code designed for GCC (32 bits) + asm_vc_64 - with asm for VC (64 bit) + asm_gcc_64 - with asm for GCC (64 bit) + no_asm_32 - pure C++ version (32 bit) - without any asm code + no_asm_64 - pure C++ version (64 bit) - without any asm code + */ + template + const char * UInt::LibTypeStr() + { + #ifndef __GNUC__ + static const char info[] = "asm_vc_64"; + #endif + + #ifdef __GNUC__ + static const char info[] = "asm_gcc_64"; + #endif + + return info; + } + + + /*! + returning the currect type of the library + */ + template + LibTypeCode UInt::LibType() + { + #ifndef __GNUC__ + LibTypeCode info = asm_vc_64; + #endif + + #ifdef __GNUC__ + LibTypeCode info = asm_gcc_64; + #endif + + return info; + } + + + /*! + * + * basic mathematic functions + * + */ + + + + /*! + this method adding ss2 to the this and adding carry if it's defined + (this = this + ss2 + c) + + ***this method is created only on a 64bit platform*** + + c must be zero or one (might be a bigger value than 1) + function returns carry (1) (if it was) + */ + template + uint UInt::Add(const UInt & ss2, uint c) + { + uint b = value_size; + uint * p1 = table; + const uint * p2 = ss2.table; + + // we don't have to use TTMATH_REFERENCE_ASSERT here + // this algorithm doesn't require it + + #ifndef __GNUC__ + c = ttmath_adc_x64(p1,p2,b,c); + #endif + + #ifdef __GNUC__ + uint dummy, dummy2; + + /* + this part should be compiled with gcc + */ + __asm__ __volatile__( + + "xorq %%rdx, %%rdx \n" + "negq %%rax \n" // CF=1 if rax!=0 , CF=0 if rax==0 + + "1: \n" + "movq (%%rsi,%%rdx,8), %%rax \n" + "adcq %%rax, (%%rbx,%%rdx,8) \n" + + "incq %%rdx \n" + "decq %%rcx \n" + "jnz 1b \n" + + "adcq %%rcx, %%rcx \n" + + : "=c" (c), "=a" (dummy), "=d" (dummy2) + : "0" (b), "1" (c), "b" (p1), "S" (p2) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::Add", c) + + return c; + } + + + + /*! + this method adds one word (at a specific position) + and returns a carry (if it was) + + ***this method is created only on a 64bit platform*** + + + if we've got (value_size=3): + table[0] = 10; + table[1] = 30; + table[2] = 5; + and we call: + AddInt(2,1) + then it'll be: + table[0] = 10; + table[1] = 30 + 2; + table[2] = 5; + + of course if there was a carry from table[2] it would be returned + */ + template + uint UInt::AddInt(uint value, uint index) + { + uint b = value_size; + uint * p1 = table; + uint c; + + TTMATH_ASSERT( index < value_size ) + + #ifndef __GNUC__ + c = ttmath_addindexed_x64(p1,b,index,value); + #endif + + + #ifdef __GNUC__ + uint dummy, dummy2; + + __asm__ __volatile__( + + "subq %%rdx, %%rcx \n" + + "1: \n" + "addq %%rax, (%%rbx,%%rdx,8) \n" + "jnc 2f \n" + + "movq $1, %%rax \n" + "incq %%rdx \n" + "decq %%rcx \n" + "jnz 1b \n" + + "2: \n" + "setc %%al \n" + "movzx %%al, %%rdx \n" + + : "=d" (c), "=a" (dummy), "=c" (dummy2) + : "0" (index), "1" (value), "2" (b), "b" (p1) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::AddInt", c) + + return c; + } + + + + /*! + this method adds only two unsigned words to the existing value + and these words begin on the 'index' position + (it's used in the multiplication algorithm 2) + + ***this method is created only on a 64bit platform*** + + index should be equal or smaller than value_size-2 (index <= value_size-2) + x1 - lower word, x2 - higher word + + for example if we've got value_size equal 4 and: + table[0] = 3 + table[1] = 4 + table[2] = 5 + table[3] = 6 + then let + x1 = 10 + x2 = 20 + and + index = 1 + + the result of this method will be: + table[0] = 3 + table[1] = 4 + x1 = 14 + table[2] = 5 + x2 = 25 + table[3] = 6 + + and no carry at the end of table[3] + + (of course if there was a carry in table[2](5+20) then + this carry would be passed to the table[3] etc.) + */ + template + uint UInt::AddTwoInts(uint x2, uint x1, uint index) + { + uint b = value_size; + uint * p1 = table; + uint c; + + TTMATH_ASSERT( index < value_size - 1 ) + + #ifndef __GNUC__ + c = ttmath_addindexed2_x64(p1,b,index,x1,x2); + #endif + + + #ifdef __GNUC__ + uint dummy, dummy2; + + __asm__ __volatile__( + + "subq %%rdx, %%rcx \n" + + "addq %%rsi, (%%rbx,%%rdx,8) \n" + "incq %%rdx \n" + "decq %%rcx \n" + + "1: \n" + "adcq %%rax, (%%rbx,%%rdx,8) \n" + "jnc 2f \n" + + "mov $0, %%rax \n" + "incq %%rdx \n" + "decq %%rcx \n" + "jnz 1b \n" + + "2: \n" + "setc %%al \n" + "movzx %%al, %%rax \n" + + : "=a" (c), "=c" (dummy), "=d" (dummy2) + : "0" (x2), "1" (b), "2" (index), "b" (p1), "S" (x1) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::AddTwoInts", c) + + return c; + } + + + + /*! + this static method addes one vector to the other + 'ss1' is larger in size or equal to 'ss2' + + ss1 points to the first (larger) vector + ss2 points to the second vector + ss1_size - size of the ss1 (and size of the result too) + ss2_size - size of the ss2 + result - is the result vector (which has size the same as ss1: ss1_size) + + Example: ss1_size is 5, ss2_size is 3 + ss1: ss2: result (output): + 5 1 5+1 + 4 3 4+3 + 2 7 2+7 + 6 6 + 9 9 + of course the carry is propagated and will be returned from the last item + (this method is used by the Karatsuba multiplication algorithm) + */ + template + uint UInt::AddVector(const uint * ss1, const uint * ss2, uint ss1_size, uint ss2_size, uint * result) + { + TTMATH_ASSERT( ss1_size >= ss2_size ) + + uint c; + + #ifndef __GNUC__ + c = ttmath_addvector_x64(ss1, ss2, ss1_size, ss2_size, result); + #endif + + + #ifdef __GNUC__ + uint dummy1, dummy2, dummy3; + uint rest = ss1_size - ss2_size; + + // this part should be compiled with gcc + + __asm__ __volatile__( + "mov %%rdx, %%r8 \n" + "xor %%rdx, %%rdx \n" // rdx = 0, cf = 0 + "1: \n" + "mov (%%rsi,%%rdx,8), %%rax \n" + "adc (%%rbx,%%rdx,8), %%rax \n" + "mov %%rax, (%%rdi,%%rdx,8) \n" + + "inc %%rdx \n" + "dec %%rcx \n" + "jnz 1b \n" + + "adc %%rcx, %%rcx \n" // rcx has the cf state + + "or %%r8, %%r8 \n" + "jz 3f \n" + + "xor %%rbx, %%rbx \n" // ebx = 0 + "neg %%rcx \n" // setting cf from rcx + "mov %%r8, %%rcx \n" // rcx=rest and is != 0 + "2: \n" + "mov (%%rsi, %%rdx, 8), %%rax \n" + "adc %%rbx, %%rax \n" + "mov %%rax, (%%rdi, %%rdx, 8) \n" + + "inc %%rdx \n" + "dec %%rcx \n" + "jnz 2b \n" + + "adc %%rcx, %%rcx \n" + "3: \n" + + : "=a" (dummy1), "=b" (dummy2), "=c" (c), "=d" (dummy3) + : "1" (ss2), "2" (ss2_size), "3" (rest), "S" (ss1), "D" (result) + : "%r8", "cc", "memory" ); + + #endif + + TTMATH_VECTOR_LOGC("UInt::AddVector", c, result, ss1_size) + + return c; + } + + + + /*! + this method's subtracting ss2 from the 'this' and subtracting + carry if it has been defined + (this = this - ss2 - c) + + ***this method is created only on a 64bit platform*** + + c must be zero or one (might be a bigger value than 1) + function returns carry (1) (if it was) + */ + template + uint UInt::Sub(const UInt & ss2, uint c) + { + uint b = value_size; + uint * p1 = table; + const uint * p2 = ss2.table; + + // we don't have to use TTMATH_REFERENCE_ASSERT here + // this algorithm doesn't require it + + #ifndef __GNUC__ + c = ttmath_sbb_x64(p1,p2,b,c); + #endif + + + #ifdef __GNUC__ + uint dummy, dummy2; + + __asm__ __volatile__( + + "xorq %%rdx, %%rdx \n" + "negq %%rax \n" // CF=1 if rax!=0 , CF=0 if rax==0 + + "1: \n" + "movq (%%rsi,%%rdx,8), %%rax \n" + "sbbq %%rax, (%%rbx,%%rdx,8) \n" + + "incq %%rdx \n" + "decq %%rcx \n" + "jnz 1b \n" + + "adcq %%rcx, %%rcx \n" + + : "=c" (c), "=a" (dummy), "=d" (dummy2) + : "0" (b), "1" (c), "b" (p1), "S" (p2) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::Sub", c) + + return c; + } + + + + /*! + this method subtracts one word (at a specific position) + and returns a carry (if it was) + + ***this method is created only on a 64bit platform*** + + if we've got (value_size=3): + table[0] = 10; + table[1] = 30; + table[2] = 5; + and we call: + SubInt(2,1) + then it'll be: + table[0] = 10; + table[1] = 30 - 2; + table[2] = 5; + + of course if there was a carry from table[2] it would be returned + */ + template + uint UInt::SubInt(uint value, uint index) + { + uint b = value_size; + uint * p1 = table; + uint c; + + TTMATH_ASSERT( index < value_size ) + + #ifndef __GNUC__ + c = ttmath_subindexed_x64(p1,b,index,value); + #endif + + + #ifdef __GNUC__ + uint dummy, dummy2; + + __asm__ __volatile__( + + "subq %%rdx, %%rcx \n" + + "1: \n" + "subq %%rax, (%%rbx,%%rdx,8) \n" + "jnc 2f \n" + + "movq $1, %%rax \n" + "incq %%rdx \n" + "decq %%rcx \n" + "jnz 1b \n" + + "2: \n" + "setc %%al \n" + "movzx %%al, %%rdx \n" + + : "=d" (c), "=a" (dummy), "=c" (dummy2) + : "0" (index), "1" (value), "2" (b), "b" (p1) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::SubInt", c) + + return c; + } + + + /*! + this static method subtractes one vector from the other + 'ss1' is larger in size or equal to 'ss2' + + ss1 points to the first (larger) vector + ss2 points to the second vector + ss1_size - size of the ss1 (and size of the result too) + ss2_size - size of the ss2 + result - is the result vector (which has size the same as ss1: ss1_size) + + Example: ss1_size is 5, ss2_size is 3 + ss1: ss2: result (output): + 5 1 5-1 + 4 3 4-3 + 2 7 2-7 + 6 6-1 (the borrow from previous item) + 9 9 + return (carry): 0 + of course the carry (borrow) is propagated and will be returned from the last item + (this method is used by the Karatsuba multiplication algorithm) + */ + template + uint UInt::SubVector(const uint * ss1, const uint * ss2, uint ss1_size, uint ss2_size, uint * result) + { + TTMATH_ASSERT( ss1_size >= ss2_size ) + + uint c; + + #ifndef __GNUC__ + c = ttmath_subvector_x64(ss1, ss2, ss1_size, ss2_size, result); + #endif + + + #ifdef __GNUC__ + + // the asm code is nearly the same as in AddVector + // only two instructions 'adc' are changed to 'sbb' + + uint dummy1, dummy2, dummy3; + uint rest = ss1_size - ss2_size; + + __asm__ __volatile__( + "mov %%rdx, %%r8 \n" + "xor %%rdx, %%rdx \n" // rdx = 0, cf = 0 + "1: \n" + "mov (%%rsi,%%rdx,8), %%rax \n" + "sbb (%%rbx,%%rdx,8), %%rax \n" + "mov %%rax, (%%rdi,%%rdx,8) \n" + + "inc %%rdx \n" + "dec %%rcx \n" + "jnz 1b \n" + + "adc %%rcx, %%rcx \n" // rcx has the cf state + + "or %%r8, %%r8 \n" + "jz 3f \n" + + "xor %%rbx, %%rbx \n" // ebx = 0 + "neg %%rcx \n" // setting cf from rcx + "mov %%r8, %%rcx \n" // rcx=rest and is != 0 + "2: \n" + "mov (%%rsi, %%rdx, 8), %%rax \n" + "sbb %%rbx, %%rax \n" + "mov %%rax, (%%rdi, %%rdx, 8) \n" + + "inc %%rdx \n" + "dec %%rcx \n" + "jnz 2b \n" + + "adc %%rcx, %%rcx \n" + "3: \n" + + : "=a" (dummy1), "=b" (dummy2), "=c" (c), "=d" (dummy3) + : "1" (ss2), "2" (ss2_size), "3" (rest), "S" (ss1), "D" (result) + : "%r8", "cc", "memory" ); + + #endif + + TTMATH_VECTOR_LOGC("UInt::SubVector", c, result, ss1_size) + + return c; + } + + + /*! + this method moves all bits into the left hand side + return value <- this <- c + + the lowest *bit* will be held the 'c' and + the state of one additional bit (on the left hand side) + will be returned + + for example: + let this is 001010000 + after Rcl2_one(1) there'll be 010100001 and Rcl2_one returns 0 + + ***this method is created only on a 64bit platform*** + */ + template + uint UInt::Rcl2_one(uint c) + { + sint b = value_size; + uint * p1 = table; + + + #ifndef __GNUC__ + c = ttmath_rcl_x64(p1,b,c); + #endif + + + #ifdef __GNUC__ + uint dummy, dummy2; + + __asm__ __volatile__( + + "xorq %%rdx, %%rdx \n" // rdx=0 + "negq %%rax \n" // CF=1 if rax!=0 , CF=0 if rax==0 + + "1: \n" + "rclq $1, (%%rbx, %%rdx, 8) \n" + + "incq %%rdx \n" + "decq %%rcx \n" + "jnz 1b \n" + + "adcq %%rcx, %%rcx \n" + + : "=c" (c), "=a" (dummy), "=d" (dummy2) + : "0" (b), "1" (c), "b" (p1) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::Rcl2_one", c) + + return c; + } + + + /*! + this method moves all bits into the right hand side + c -> this -> return value + + the highest *bit* will be held the 'c' and + the state of one additional bit (on the right hand side) + will be returned + + for example: + let this is 000000010 + after Rcr2_one(1) there'll be 100000001 and Rcr2_one returns 0 + + ***this method is created only on a 64bit platform*** + */ + template + uint UInt::Rcr2_one(uint c) + { + sint b = value_size; + uint * p1 = table; + + + #ifndef __GNUC__ + c = ttmath_rcr_x64(p1,b,c); + #endif + + + #ifdef __GNUC__ + uint dummy; + + __asm__ __volatile__( + + "negq %%rax \n" // CF=1 if rax!=0 , CF=0 if rax==0 + + "1: \n" + "rcrq $1, -8(%%rbx, %%rcx, 8) \n" + + "decq %%rcx \n" + "jnz 1b \n" + + "adcq %%rcx, %%rcx \n" + + : "=c" (c), "=a" (dummy) + : "0" (b), "1" (c), "b" (p1) + : "cc", "memory" ); + + #endif + + TTMATH_LOGC("UInt::Rcr2_one", c) + + return c; + } + + + + /*! + this method moves all bits into the left hand side + return value <- this <- c + + the lowest *bits* will be held the 'c' and + the state of one additional bit (on the left hand side) + will be returned + + for example: + let this is 001010000 + after Rcl2(3, 1) there'll be 010000111 and Rcl2 returns 1 + + ***this method is created only on a 64bit platform*** + */ + template + uint UInt::Rcl2(uint bits, uint c) + { + TTMATH_ASSERT( bits>0 && bits this -> return value + + the highest *bits* will be held the 'c' and + the state of one additional bit (on the right hand side) + will be returned + + for example: + let this is 000000010 + after Rcr2(2, 1) there'll be 110000000 and Rcr2 returns 1 + + ***this method is created only on a 64bit platform*** + */ + template + uint UInt::Rcr2(uint bits, uint c) + { + TTMATH_ASSERT( bits>0 && bits + sint UInt::FindLeadingBitInWord(uint x) + { + sint result; + + + #ifndef __GNUC__ + + unsigned long nIndex = 0; + + if( _BitScanReverse64(&nIndex,x) == 0 ) + result = -1; + else + result = nIndex; + + #endif + + + #ifdef __GNUC__ + uint dummy; + + __asm__ ( + + "movq $-1, %1 \n" + "bsrq %2, %0 \n" + "cmovz %1, %0 \n" + + : "=r" (result), "=&r" (dummy) + : "r" (x) + : "cc" ); + + #endif + + + return result; + } + + + /* + this method returns the number of the highest set bit in one 64-bit word + if the 'x' is zero this method returns '-1' + + ***this method is created only on a 64bit platform*** + */ + template + sint UInt::FindLowestBitInWord(uint x) + { + sint result; + + + #ifndef __GNUC__ + + unsigned long nIndex = 0; + + if( _BitScanForward64(&nIndex,x) == 0 ) + result = -1; + else + result = nIndex; + + #endif + + + #ifdef __GNUC__ + uint dummy; + + __asm__ ( + + "movq $-1, %1 \n" + "bsfq %2, %0 \n" + "cmovz %1, %0 \n" + + : "=r" (result), "=&r" (dummy) + : "r" (x) + : "cc" ); + + #endif + + + return result; + } + + + /*! + this method sets a special bit in the 'value' + and returns the last state of the bit (zero or one) + + ***this method is created only on a 64bit platform*** + + bit is from <0,63> + + e.g. + uint x = 100; + uint bit = SetBitInWord(x, 3); + now: x = 108 and bit = 0 + */ + template + uint UInt::SetBitInWord(uint & value, uint bit) + { + TTMATH_ASSERT( bit < TTMATH_BITS_PER_UINT ) + + uint old_bit; + uint v = value; + + + #ifndef __GNUC__ + old_bit = _bittestandset64((__int64*)&value,bit) != 0; + #endif + + + #ifdef __GNUC__ + + __asm__ ( + + "btsq %%rbx, %%rax \n" + "setc %%bl \n" + "movzx %%bl, %%rbx \n" + + : "=a" (v), "=b" (old_bit) + : "0" (v), "1" (bit) + : "cc" ); + + #endif + + value = v; + + return old_bit; + } + + + /*! + * + * Multiplication + * + * + */ + + + /*! + multiplication: result_high:result_low = a * b + result_high - higher word of the result + result_low - lower word of the result + + this methos never returns a carry + this method is used in the second version of the multiplication algorithms + + ***this method is created only on a 64bit platform*** + */ + template + void UInt::MulTwoWords(uint a, uint b, uint * result_high, uint * result_low) + { + /* + we must use these temporary variables in order to inform the compilator + that value pointed with result1 and result2 has changed + + this has no effect in visual studio but it's usefull when + using gcc and options like -O + */ + uint result1_; + uint result2_; + + + #ifndef __GNUC__ + result1_ = _umul128(a,b,&result2_); + #endif + + + #ifdef __GNUC__ + + __asm__ ( + + "mulq %%rdx \n" + + : "=a" (result1_), "=d" (result2_) + : "0" (a), "1" (b) + : "cc" ); + + #endif + + + *result_low = result1_; + *result_high = result2_; + } + + + + + /*! + * + * Division + * + * + */ + + + /*! + this method calculates 64bits word a:b / 32bits c (a higher, b lower word) + r = a:b / c and rest - remainder + + ***this method is created only on a 64bit platform*** + + * + * WARNING: + * if r (one word) is too small for the result or c is equal zero + * there'll be a hardware interruption (0) + * and probably the end of your program + * + */ + template + void UInt::DivTwoWords(uint a,uint b, uint c, uint * r, uint * rest) + { + uint r_; + uint rest_; + /* + these variables have similar meaning like those in + the multiplication algorithm MulTwoWords + */ + + TTMATH_ASSERT( c != 0 ) + + + #ifndef __GNUC__ + + ttmath_div_x64(&a,&b,c); + r_ = a; + rest_ = b; + + #endif + + + #ifdef __GNUC__ + + __asm__ ( + + "divq %%rcx \n" + + : "=a" (r_), "=d" (rest_) + : "d" (a), "a" (b), "c" (c) + : "cc" ); + + #endif + + + *r = r_; + *rest = rest_; + } + +} //namespace + + +#endif //ifdef TTMATH_PLATFORM64 +#endif //ifndef TTMATH_NOASM +#endif + + diff --git a/extern/ttmath/ttmathuint_x86_64_msvc.asm b/extern/ttmath/ttmathuint_x86_64_msvc.asm new file mode 100644 index 0000000000..b7c85c2b84 --- /dev/null +++ b/extern/ttmath/ttmathuint_x86_64_msvc.asm @@ -0,0 +1,548 @@ +; +; This file is a part of TTMath Bignum Library +; and is distributed under the (new) BSD licence. +; Author: Christian Kaiser +; + +; +; Copyright (c) 2009, Christian Kaiser +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; * Redistributions of source code must retain the above copyright notice, +; this list of conditions and the following disclaimer. +; +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in the +; documentation and/or other materials provided with the distribution. +; +; * Neither the name Christian Kaiser nor the names of contributors to this +; project may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +; ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +; LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +; CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +; SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +; INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +; CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +; ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +; THE POSSIBILITY OF SUCH DAMAGE. +; + +; +; compile with debug info: ml64.exe /c /Zd /Zi ttmathuint_x86_64_msvc.asm +; compile without debug info: ml64.exe /c ttmathuint_x86_64_msvc.asm +; this creates ttmathuint_x86_64_msvc.obj file which can be linked with your program +; + +PUBLIC ttmath_adc_x64 +PUBLIC ttmath_addindexed_x64 +PUBLIC ttmath_addindexed2_x64 +PUBLIC ttmath_addvector_x64 + +PUBLIC ttmath_sbb_x64 +PUBLIC ttmath_subindexed_x64 +PUBLIC ttmath_subvector_x64 + +PUBLIC ttmath_rcl_x64 +PUBLIC ttmath_rcr_x64 + +PUBLIC ttmath_rcl2_x64 +PUBLIC ttmath_rcr2_x64 + +PUBLIC ttmath_div_x64 + +; +; Microsoft x86_64 convention: http://msdn.microsoft.com/en-us/library/9b372w95.aspx +; +; "rax, rcx, rdx, r8-r11 are volatile." +; "rbx, rbp, rdi, rsi, r12-r15 are nonvolatile." +; + + +.CODE + + + ALIGN 8 + +;---------------------------------------- + +ttmath_adc_x64 PROC + ; rcx = p1 + ; rdx = p2 + ; r8 = nSize + ; r9 = nCarry + + xor rax, rax + xor r11, r11 + sub rax, r9 ; sets CARRY if r9 != 0 + + ALIGN 16 + loop1: + mov rax,qword ptr [rdx + r11 * 8] + adc qword ptr [rcx + r11 * 8], rax + lea r11, [r11+1] + dec r8 + jnz loop1 + + setc al + movzx rax, al + + ret + +ttmath_adc_x64 ENDP + +;---------------------------------------- + + ALIGN 8 + +;---------------------------------------- + +ttmath_addindexed_x64 PROC + + ; rcx = p1 + ; rdx = nSize + ; r8 = nPos + ; r9 = nValue + + xor rax, rax ; rax = result + sub rdx, r8 ; rdx = remaining count of uints + + add qword ptr [rcx + r8 * 8], r9 + jc next1 + + ret + +next1: + mov r9, 1 + + ALIGN 16 +loop1: + dec rdx + jz done_with_cy + lea r8, [r8+1] + add qword ptr [rcx + r8 * 8], r9 + jc loop1 + + ret + +done_with_cy: + lea rax, [rax+1] ; rax = 1 + + ret + +ttmath_addindexed_x64 ENDP + +;---------------------------------------- + + ALIGN 8 + +;---------------------------------------- + +ttmath_addindexed2_x64 PROC + + ; rcx = p1 (pointer) + ; rdx = b (value size) + ; r8 = nPos + ; r9 = nValue1 + ; [esp+0x28] = nValue2 + + xor rax, rax ; return value + mov r11, rcx ; table + sub rdx, r8 ; rdx = remaining count of uints + mov r10, [esp+028h] ; r10 = nValue2 + + add qword ptr [r11 + r8 * 8], r9 + lea r8, [r8+1] + lea rdx, [rdx-1] + adc qword ptr [r11 + r8 * 8], r10 + jc next + ret + + ALIGN 16 +loop1: + lea r8, [r8+1] + add qword ptr [r11 + r8 * 8], 1 + jc next + ret + +next: + dec rdx ; does not modify CY too... + jnz loop1 + lea rax, [rax+1] + ret + +ttmath_addindexed2_x64 ENDP + + + +;---------------------------------------- + + ALIGN 8 + +;---------------------------------------- + + +ttmath_addvector_x64 PROC + ; rcx = ss1 + ; rdx = ss2 + ; r8 = ss1_size + ; r9 = ss2_size + ; [esp+0x28] = result + + mov r10, [esp+028h] + sub r8, r9 + xor r11, r11 ; r11=0, cf=0 + + ALIGN 16 + loop1: + mov rax, qword ptr [rcx + r11 * 8] + adc rax, qword ptr [rdx + r11 * 8] + mov qword ptr [r10 + r11 * 8], rax + inc r11 + dec r9 + jnz loop1 + + adc r9, r9 ; r9 has the cf state + + or r8, r8 + jz done + + neg r9 ; setting cf from r9 + mov r9, 0 ; don't use xor here (cf is used) + loop2: + mov rax, qword ptr [rcx + r11 * 8] + adc rax, r9 + mov qword ptr [r10 + r11 * 8], rax + inc r11 + dec r8 + jnz loop2 + + adc r8, r8 + mov rax, r8 + + ret + +done: + mov rax, r9 + ret + +ttmath_addvector_x64 ENDP + + +;---------------------------------------- + + ALIGN 8 + +;---------------------------------------- + +ttmath_sbb_x64 PROC + + ; rcx = p1 + ; rdx = p2 + ; r8 = nCount + ; r9 = nCarry + + xor rax, rax + xor r11, r11 + sub rax, r9 ; sets CARRY if r9 != 0 + + ALIGN 16 + loop1: + mov rax,qword ptr [rdx + r11 * 8] + sbb qword ptr [rcx + r11 * 8], rax + lea r11, [r11+1] + dec r8 + jnz loop1 + + setc al + movzx rax, al + + ret + +ttmath_sbb_x64 ENDP + +;---------------------------------------- + + ALIGN 8 + +;---------------------------------------- + +ttmath_subindexed_x64 PROC + ; rcx = p1 + ; rdx = nSize + ; r8 = nPos + ; r9 = nValue + + sub rdx, r8 ; rdx = remaining count of uints + + ALIGN 16 +loop1: + sub qword ptr [rcx + r8 * 8], r9 + jnc done + + lea r8, [r8+1] + mov r9, 1 + dec rdx + jnz loop1 + + mov rax, 1 + ret + +done: + xor rax, rax + ret + +ttmath_subindexed_x64 ENDP + + + +;---------------------------------------- + + ALIGN 8 + +;---------------------------------------- + +; the same asm code as in addvector_x64 only two instructions 'adc' changed to 'sbb' + +ttmath_subvector_x64 PROC + ; rcx = ss1 + ; rdx = ss2 + ; r8 = ss1_size + ; r9 = ss2_size + ; [esp+0x28] = result + + mov r10, [esp+028h] + sub r8, r9 + xor r11, r11 ; r11=0, cf=0 + + ALIGN 16 + loop1: + mov rax, qword ptr [rcx + r11 * 8] + sbb rax, qword ptr [rdx + r11 * 8] + mov qword ptr [r10 + r11 * 8], rax + inc r11 + dec r9 + jnz loop1 + + adc r9, r9 ; r9 has the cf state + + or r8, r8 + jz done + + neg r9 ; setting cf from r9 + mov r9, 0 ; don't use xor here (cf is used) + loop2: + mov rax, qword ptr [rcx + r11 * 8] + sbb rax, r9 + mov qword ptr [r10 + r11 * 8], rax + inc r11 + dec r8 + jnz loop2 + + adc r8, r8 + mov rax, r8 + + ret + +done: + mov rax, r9 + ret + +ttmath_subvector_x64 ENDP + + + + +;---------------------------------------- + + ALIGN 8 + +;---------------------------------------- + +ttmath_rcl_x64 PROC + ; rcx = p1 + ; rdx = b + ; r8 = nLowestBit + + mov r11, rcx ; table + xor r10, r10 + neg r8 ; CY set if r8 <> 0 + + ALIGN 16 +loop1: + rcl qword ptr [r11 + r10 * 8], 1 + lea r10, [r10+1] + dec rdx + jnz loop1 + + setc al + movzx rax, al + + ret + +ttmath_rcl_x64 ENDP + +;---------------------------------------- + + ALIGN 8 + +;---------------------------------------- + +ttmath_rcr_x64 PROC + ; rcx = p1 + ; rdx = nSize + ; r8 = nLowestBit + + xor r10, r10 + neg r8 ; CY set if r8 <> 0 + + ALIGN 16 +loop1: + rcr qword ptr -8[rcx + rdx * 8], 1 + dec rdx + jnz loop1 + + setc al + movzx rax, al + + ret + +ttmath_rcr_x64 ENDP + +;---------------------------------------- + + ALIGN 8 + +;---------------------------------------- + +ttmath_div_x64 PROC + + ; rcx = &Hi + ; rdx = &Lo + ; r8 = nDiv + + mov r11, rcx + mov r10, rdx + + mov rdx, qword ptr [r11] + mov rax, qword ptr [r10] + div r8 + mov qword ptr [r10], rdx ; remainder + mov qword ptr [r11], rax ; value + + ret + +ttmath_div_x64 ENDP + +;---------------------------------------- + + ALIGN 8 + +;---------------------------------------- + +ttmath_rcl2_x64 PROC + ; rcx = p1 + ; rdx = nSize + ; r8 = bits + ; r9 = c + + push rbx + + mov r10, rcx ; r10 = p1 + xor rax, rax + + mov rcx, 64 + sub rcx, r8 + + mov r11, -1 + shr r11, cl ; r11 = mask + + mov rcx, r8 ; rcx = count of bits + + mov rbx, rax ; rbx = old value = 0 + or r9, r9 + cmovnz rbx, r11 ; if (c) then old value = mask + + mov r9, rax ; r9 = index (0..nSize-1) + + ALIGN 16 +loop1: + rol qword ptr [r10+r9*8], cl + mov rax, qword ptr [r10+r9*8] + and rax, r11 + xor qword ptr [r10+r9*8], rax + or qword ptr [r10+r9*8], rbx + mov rbx, rax + + lea r9, [r9+1] + dec rdx + + jnz loop1 + + and rax, 1 + pop rbx + ret + +ttmath_rcl2_x64 ENDP + +;---------------------------------------- + + ALIGN 8 + +;---------------------------------------- + +ttmath_rcr2_x64 PROC + ; rcx = p1 + ; rdx = nSize + ; r8 = bits + ; r9 = c + + push rbx + mov r10, rcx ; r10 = p1 + xor rax, rax + + mov rcx, 64 + sub rcx, r8 + + mov r11, -1 + shl r11, cl ; r11 = mask + + mov rcx, r8 ; rcx = count of bits + + mov rbx, rax ; rbx = old value = 0 + or r9, r9 + cmovnz rbx, r11 ; if (c) then old value = mask + + mov r9, rdx ; r9 = index (0..nSize-1) + lea r9, [r9-1] + + ALIGN 16 +loop1: + ror qword ptr [r10+r9*8], cl + mov rax, qword ptr [r10+r9*8] + and rax, r11 + xor qword ptr [r10+r9*8], rax + or qword ptr [r10+r9*8], rbx + mov rbx, rax + + lea r9, [r9-1] + dec rdx + + jnz loop1 + + rol rax, 1 + and rax, 1 + pop rbx + + ret + +ttmath_rcr2_x64 ENDP + +END diff --git a/src/burp/backup.epp b/src/burp/backup.epp index 43874a4c1d..b31f1383ed 100644 --- a/src/burp/backup.epp +++ b/src/burp/backup.epp @@ -1472,8 +1472,8 @@ void put_data(burp_rel* relation) add_byte(blr, field->fld_scale); break; - case blr_dec_fixed: - alignment = type_alignments[dtype_dec_fixed]; + case blr_int128: + alignment = type_alignments[dtype_int128]; add_byte(blr, field->fld_type); add_byte(blr, field->fld_scale); break; diff --git a/src/burp/canonical.cpp b/src/burp/canonical.cpp index ff7404de3f..7923d3de40 100644 --- a/src/burp/canonical.cpp +++ b/src/burp/canonical.cpp @@ -170,11 +170,15 @@ ULONG CAN_encode_decode(burp_rel* relation, lstring* buffer, UCHAR* data, bool d break; case dtype_dec128: - case dtype_dec_fixed: if (!xdr_dec128(xdrs, (Firebird::Decimal128*) p)) return FALSE; break; + case dtype_int128: + if (!xdr_int128(xdrs, (Firebird::Int128*) p)) + return FALSE; + break; + case dtype_timestamp: if (!xdr_long(xdrs, &((SLONG*) p)[0])) return FALSE; diff --git a/src/burp/restore.epp b/src/burp/restore.epp index 7870ee7a0f..6f030b195b 100644 --- a/src/burp/restore.epp +++ b/src/burp/restore.epp @@ -3579,7 +3579,7 @@ rec_type get_data_old(BurpGlobals* tdgbl, burp_rel* relation) case blr_long: case blr_quad: case blr_int64: - case blr_dec_fixed: + case blr_int128: add_byte(blr, field->fld_type); add_byte(blr, field->fld_scale); break; diff --git a/src/common/DecFloat.cpp b/src/common/DecFloat.cpp index 9062bc68f5..75d330119a 100644 --- a/src/common/DecFloat.cpp +++ b/src/common/DecFloat.cpp @@ -28,6 +28,7 @@ #include "firebird.h" #include "DecFloat.h" +#include "Int128.h" #include "StatusArg.h" #include "gen/iberror.h" @@ -49,7 +50,7 @@ extern "C" using namespace Firebird; const DecimalStatus DecimalStatus::DEFAULT(FB_DEC_Errors); -const DecimalBinding DecimalBinding::DEFAULT; +const NumericBinding NumericBinding::DEFAULT; namespace { @@ -77,7 +78,7 @@ public: init(DEC_INIT_DECIMAL64); } - DecimalContext(const Decimal128Base*, DecimalStatus ds) + DecimalContext(const Decimal128*, DecimalStatus ds) : decSt(ds) { init(DEC_INIT_DECIMAL128); @@ -126,6 +127,8 @@ const CDecimal128 dmax(DBL_MAX, DecimalStatus(0)), dmin(-DBL_MAX, DecimalStatus( const CDecimal128 dzup(DBL_MIN, DecimalStatus(0)), dzlw(-DBL_MIN, DecimalStatus(0)); const CDecimal128 i64max(MAX_SINT64, DecimalStatus(0)), i64min(MIN_SINT64, DecimalStatus(0)); const CDecimal128 c1(1); +const CDecimal128 pow2_32("4294967296", DecimalStatus(0)); +const CDecimal128 pow2_64("18446744073709551616", DecimalStatus(0)); unsigned digits(const unsigned pMax, unsigned char* const coeff, int& exp) { @@ -265,7 +268,7 @@ Decimal64 Decimal64::set(SLONG value, DecimalStatus decSt, int scale) return *this; } -Decimal64 Decimal64::set(DecimalFixed value, DecimalStatus decSt, int scale) +Decimal64 Decimal64::set(Int128 value, DecimalStatus decSt, int scale) { Decimal128 tmp; tmp.set(value, decSt, scale); @@ -518,11 +521,21 @@ Decimal128 Decimal128::set(SLONG value, DecimalStatus decSt, int scale) return *this; } -Decimal128 Decimal128::set(DecimalFixed value, DecimalStatus decSt, int scale) +Decimal128 Decimal128::set(Int128 value, DecimalStatus decSt, int scale) { - *this = value; - setScale(decSt, -scale); + unsigned dwords[4]; + value.getTable32(dwords); + DecimalContext context(this, decSt); + decQuadFromInt32(&dec, dwords[3]); + for (int i = 3; i--; ) + { + decQuad dw; + decQuadFromUInt32(&dw, dwords[i]); + decQuadFMA(&dec, &dec, &pow2_32.dec, &dw, &context); + } + + setScale(decSt, -scale); return *this; } @@ -533,13 +546,10 @@ Decimal128 Decimal128::set(SINT64 value, DecimalStatus decSt, int scale) unsigned low = value & 0xFFFFFFFF; DecimalContext context(this, decSt); - decQuad pow2_32; - decQuadFromString(&pow2_32, "4294967296", &context); - decQuad up, down; decQuadFromInt32(&up, high); decQuadFromUInt32(&down, low); - decQuadFMA(&dec, &up, &pow2_32, &down, &context); + decQuadFMA(&dec, &up, &pow2_32.dec, &down, &context); } setScale(decSt, -scale); @@ -565,80 +575,6 @@ Decimal128 Decimal128::set(double value, DecimalStatus decSt) return *this; } -DecimalFixed DecimalFixed::set(SLONG value) -{ - decQuadFromInt32(&dec, value); - return *this; -} - -DecimalFixed DecimalFixed::set(SINT64 value) -{ - int high = value >> 32; - unsigned low = value & 0xFFFFFFFF; - - DecimalContext context(this, DecimalStatus(0)); - decQuad pow2_32; - decQuadFromString(&pow2_32, "4294967296", &context); - - decQuad up, down; - decQuadFromInt32(&up, high); - decQuadFromUInt32(&down, low); - decQuadFMA(&dec, &up, &pow2_32, &down, &context); - - return *this; -} - -DecimalFixed DecimalFixed::set(const char* value, int scale, DecimalStatus decSt) -{ - { // scope for 'context' - DecimalContext context(this, decSt); - decQuadFromString(&dec, value, &context); - } - - exactInt(decSt, scale); - return *this; -} - -DecimalFixed DecimalFixed::set(double value, int scale, DecimalStatus decSt) -{ - char s[50]; - sprintf(s, "%18.016e", value); - { // scope for 'context' - DecimalContext context(this, decSt); - decQuadFromString(&dec, s, &context); - } - - exactInt(decSt, scale); - return *this; -} - -void DecimalFixed::exactInt(DecimalStatus decSt, int scale) -{ - setScale(decSt, -scale); - - try - { - DecimalContext context(this, decSt); - decQuadToIntegralExact(&dec, &dec, &context); - decQuadQuantize(&dec, &dec, &c1.dec, &context); - } - catch (const Exception& ex) - { - FbLocalStatus st; - ex.stuffException(&st); - - switch (st->getErrors()[1]) - { - case isc_decfloat_invalid_operation: - (Arg::Gds(isc_decfloat_invalid_operation) << - Arg::Gds(isc_numeric_out_of_range)).raise(); - break; - } - - throw; - } -} - Decimal128 Decimal128::operator=(Decimal64 d64) { decDoubleToWider(&d64.dec, &dec); @@ -654,13 +590,6 @@ int Decimal128::toInteger(DecimalStatus decSt, int scale) const return decQuadToInt32(&tmp.dec, &context, rMode); } -int DecimalFixed::toInteger(DecimalStatus decSt) const -{ - DecimalContext context(this, decSt); - enum rounding rMode = decContextGetRounding(&context); - return decQuadToInt32(&dec, &context, rMode); -} - void Decimal128::toString(DecimalStatus decSt, unsigned length, char* to) const { DecimalContext context(this, decSt); @@ -690,24 +619,7 @@ void Decimal128::toString(string& to) const to.recalculate_length(); } -Decimal128 DecimalFixed::scaled128(DecimalStatus decSt, int scale) const -{ - Decimal128 tmp; - tmp.set(*this, decSt, -scale); - return tmp; -} - -void DecimalFixed::toString(DecimalStatus decSt, int scale, unsigned length, char* to) const -{ - scaled128(decSt, scale).toString(decSt, length, to); -} - -void DecimalFixed::toString(DecimalStatus decSt, int scale, string& to) const -{ - scaled128(decSt, scale).toString(to); -} - -double Decimal128Base::toDouble(DecimalStatus decSt) const +double Decimal128::toDouble(DecimalStatus decSt) const { DecimalContext context(this, decSt); @@ -764,37 +676,12 @@ SINT64 Decimal128::toInt64(DecimalStatus decSt, int scale) const return rc; } -SINT64 DecimalFixed::toInt64(DecimalStatus decSt) const -{ - if (compare(decSt, i64min) < 0 || compare(decSt, i64max) > 0) - { - DecimalContext context(this, decSt); - decContextSetStatus(&context, DEC_Invalid_operation); - return 0; // in case of no trap on invalid operation - } - - unsigned char coeff[DECQUAD_Pmax]; - int sign = decQuadGetCoefficient(&dec, coeff); - SINT64 rc = 0; - - for (int i = 0; i < DECQUAD_Pmax; ++i) - { - rc *= 10; - if (sign) - rc -= coeff[i]; - else - rc += coeff[i]; - } - - return rc; -} - -UCHAR* Decimal128Base::getBytes() +UCHAR* Decimal128::getBytes() { return dec.bytes; } -Decimal64 Decimal128Base::toDecimal64(DecimalStatus decSt) const +Decimal64 Decimal128::toDecimal64(DecimalStatus decSt) const { Decimal64 rc; DecimalContext context(this, decSt); @@ -802,7 +689,7 @@ Decimal64 Decimal128Base::toDecimal64(DecimalStatus decSt) const return rc; } -void Decimal128Base::setScale(DecimalStatus decSt, int scale) +void Decimal128::setScale(DecimalStatus decSt, int scale) { if (scale) { @@ -812,7 +699,7 @@ void Decimal128Base::setScale(DecimalStatus decSt, int scale) } } -int Decimal128Base::compare(DecimalStatus decSt, Decimal128Base tgt) const +int Decimal128::compare(DecimalStatus decSt, Decimal128 tgt) const { DecimalContext context(this, decSt); decQuad r; @@ -820,7 +707,7 @@ int Decimal128Base::compare(DecimalStatus decSt, Decimal128Base tgt) const return decQuadToInt32(&r, &context, DEC_ROUND_HALF_UP); } -bool Decimal128Base::isInf() const +bool Decimal128::isInf() const { switch(decQuadClass(&dec)) { @@ -832,7 +719,7 @@ bool Decimal128Base::isInf() const return false; } -bool Decimal128Base::isNan() const +bool Decimal128::isNan() const { switch(decQuadClass(&dec)) { @@ -844,7 +731,7 @@ bool Decimal128Base::isNan() const return false; } -int Decimal128Base::sign() const +int Decimal128::sign() const { if (decQuadIsZero(&dec)) return 0; @@ -870,7 +757,7 @@ Decimal128 Decimal128::floor(DecimalStatus decSt) const } #ifdef DEV_BUILD -int Decimal128Base::show() +int Decimal128::show() { decQuadShow(&dec, ""); return 0; @@ -915,50 +802,6 @@ Decimal128 Decimal128::mul(DecimalStatus decSt, Decimal128 op2) const return rc; } -DecimalFixed DecimalFixed::abs() const -{ - DecimalFixed rc; - decQuadCopyAbs(&rc.dec, &dec); - return rc; -} - -DecimalFixed DecimalFixed::neg() const -{ - DecimalFixed rc; - decQuadCopyNegate(&rc.dec, &dec); - return rc; -} - -DecimalFixed DecimalFixed::add(DecimalStatus decSt, DecimalFixed op2) const -{ - DecimalContext context(this, decSt); - DecimalFixed rc; - decQuadAdd(&rc.dec, &dec, &op2.dec, &context); - context.checkForExceptions(); - decQuadQuantize(&rc.dec, &rc.dec, &c1.dec, &context); - return rc; -} - -DecimalFixed DecimalFixed::sub(DecimalStatus decSt, DecimalFixed op2) const -{ - DecimalContext context(this, decSt); - DecimalFixed rc; - decQuadSubtract(&rc.dec, &dec, &op2.dec, &context); - context.checkForExceptions(); - decQuadQuantize(&rc.dec, &rc.dec, &c1.dec, &context); - return rc; -} - -DecimalFixed DecimalFixed::mul(DecimalStatus decSt, DecimalFixed op2) const -{ - DecimalContext context(this, decSt); - DecimalFixed rc; - decQuadMultiply(&rc.dec, &dec, &op2.dec, &context); - context.checkForExceptions(); - decQuadQuantize(&rc.dec, &rc.dec, &c1.dec, &context); - return rc; -} - Decimal128 Decimal128::div(DecimalStatus decSt, Decimal128 op2) const { DecimalContext context(this, decSt); @@ -967,27 +810,6 @@ Decimal128 Decimal128::div(DecimalStatus decSt, Decimal128 op2) const return rc; } -DecimalFixed DecimalFixed::div(DecimalStatus decSt, DecimalFixed op2, int scale) const -{ - DecimalContext context(this, decSt); - DecimalFixed rc; - - // first divide with full decfloat precision - decQuadDivide(&rc.dec, &dec, &op2.dec, &context); - - // next re-scale & int-ize - rc.exactInt(decSt, scale); - return rc; -} - -DecimalFixed DecimalFixed::mod(DecimalStatus decSt, DecimalFixed op2) const -{ - DecimalContext context(this, decSt); - DecimalFixed rc; - decQuadRemainder(&rc.dec, &dec, &op2.dec, &context); - return rc; -} - Decimal128 Decimal128::fma(DecimalStatus decSt, Decimal128 op2, Decimal128 op3) const { DecimalContext context(this, decSt); @@ -1049,7 +871,7 @@ Decimal128 Decimal128::log10(DecimalStatus decSt) const return rc; } -void Decimal128Base::makeKey(ULONG* key) const +void Decimal128::makeKey(ULONG* key) const { unsigned char coeff[DECQUAD_Pmax]; int sign = decQuadGetCoefficient(&dec, coeff); @@ -1058,7 +880,7 @@ void Decimal128Base::makeKey(ULONG* key) const make(key, DECQUAD_Pmax, DECQUAD_Bias, sizeof(dec), coeff, sign, exp); } -void Decimal128Base::grabKey(ULONG* key) +void Decimal128::grabKey(ULONG* key) { int exp, sign; unsigned char bcd[DECQUAD_Pmax]; @@ -1068,12 +890,12 @@ void Decimal128Base::grabKey(ULONG* key) decQuadFromBCD(&dec, exp, bcd, sign); } -ULONG Decimal128Base::getIndexKeyLength() +ULONG Decimal128::getIndexKeyLength() { return 17; } -ULONG Decimal128Base::makeIndexKey(vary* buf) +ULONG Decimal128::makeIndexKey(vary* buf) { unsigned char coeff[DECQUAD_Pmax + 2]; int sign = decQuadGetCoefficient(&dec, coeff); @@ -1207,4 +1029,9 @@ short Decimal128::decCompare(Decimal128 op2) const return 3; } +void Decimal128::getBcd(BCD* bcd) const +{ + bcd->sign = decQuadToBCD(&dec, &bcd->exp, bcd->bcd); +} + } // namespace Firebird diff --git a/src/common/DecFloat.h b/src/common/DecFloat.h index 9cdabdfe2f..bf14abd3ab 100644 --- a/src/common/DecFloat.h +++ b/src/common/DecFloat.h @@ -115,27 +115,27 @@ struct DecimalStatus USHORT decExtFlag, roundingMode; }; -struct DecimalBinding +struct NumericBinding { enum Bind { - DEC_NATIVE, - DEC_TEXT, - DEC_DOUBLE, - DEC_NUMERIC + NUM_NATIVE, + NUM_TEXT, + NUM_DOUBLE, + NUM_INT64 }; - DecimalBinding() - : bind(DEC_NATIVE), + NumericBinding() + : bind(NUM_NATIVE), numScale(0) {} - DecimalBinding(Bind aBind, SCHAR aNumScale = 0) + NumericBinding(Bind aBind, SCHAR aNumScale = 0) : bind(aBind), numScale(aNumScale) {} - static const DecimalBinding DEFAULT; + static const NumericBinding DEFAULT; static const SCHAR MAX_SCALE = 18; Bind bind; @@ -143,13 +143,11 @@ struct DecimalBinding }; -class DecimalFixed; +class Int128; class Decimal64 { friend class Decimal128; - friend class DecimalFixed; - friend class Decimal128Base; public: #if SIZEOF_LONG < 8 @@ -159,7 +157,7 @@ public: Decimal64 set(SINT64 value, DecimalStatus decSt, int scale); Decimal64 set(const char* value, DecimalStatus decSt); Decimal64 set(double value, DecimalStatus decSt); - Decimal64 set(DecimalFixed value, DecimalStatus decSt, int scale); + Decimal64 set(Int128 value, DecimalStatus decSt, int scale); UCHAR* getBytes(); Decimal64 abs() const; @@ -193,38 +191,7 @@ private: decDouble dec; }; -class Decimal128Base -{ - friend class Decimal128; - friend class DecimalFixed; - -public: - double toDouble(DecimalStatus decSt) const; - Decimal64 toDecimal64(DecimalStatus decSt) const; - - UCHAR* getBytes(); - int compare(DecimalStatus decSt, Decimal128Base tgt) const; - - void setScale(DecimalStatus decSt, int scale); - - bool isInf() const; - bool isNan() const; - int sign() const; - - void makeKey(ULONG* key) const; - void grabKey(ULONG* key); - static ULONG getIndexKeyLength(); - ULONG makeIndexKey(vary* buf); - -#ifdef DEV_BUILD - int show(); -#endif - -private: - decQuad dec; -}; - -class Decimal128 : public Decimal128Base +class Decimal128 { friend class Decimal64; @@ -237,7 +204,7 @@ public: Decimal128 set(SINT64 value, DecimalStatus decSt, int scale); Decimal128 set(const char* value, DecimalStatus decSt); Decimal128 set(double value, DecimalStatus decSt); - Decimal128 set(DecimalFixed value, DecimalStatus decSt, int scale); + Decimal128 set(Int128 value, DecimalStatus decSt, int scale); Decimal128 operator=(Decimal64 d64); @@ -265,12 +232,37 @@ public: short totalOrder(Decimal128 op2) const; short decCompare(Decimal128 op2) const; -private: - Decimal128 operator=(Decimal128Base d128b) + double toDouble(DecimalStatus decSt) const; + Decimal64 toDecimal64(DecimalStatus decSt) const; + + UCHAR* getBytes(); + int compare(DecimalStatus decSt, Decimal128 tgt) const; + + void setScale(DecimalStatus decSt, int scale); + + bool isInf() const; + bool isNan() const; + int sign() const; + + void makeKey(ULONG* key) const; + void grabKey(ULONG* key); + static ULONG getIndexKeyLength(); + ULONG makeIndexKey(vary* buf); + +#ifdef DEV_BUILD + int show(); +#endif + + struct BCD { - memcpy(&dec, &d128b.dec, sizeof(dec)); - return *this; - } + int sign, exp; + unsigned char bcd[DECQUAD_Pmax]; + }; + + void getBcd(BCD* bcd) const; + +private: + decQuad dec; }; class CDecimal128 : public Decimal128 @@ -290,45 +282,11 @@ public: { set(value, DecimalStatus(0), 0); } -}; -class DecimalFixed : public Decimal128Base -{ -public: -#if SIZEOF_LONG < 8 - DecimalFixed set(int value) + CDecimal128(const char* value, DecimalStatus decSt) { - return set(SLONG(value)); + set(value, decSt); } -#endif - DecimalFixed set(SLONG value); - DecimalFixed set(SINT64 value); - DecimalFixed set(const char* value, int scale, DecimalStatus decSt); - DecimalFixed set(double value, int scale, DecimalStatus decSt); - - int toInteger(DecimalStatus decSt) const; - SINT64 toInt64(DecimalStatus decSt) const; - void toString(DecimalStatus decSt, int scale, unsigned length, char* to) const; - void toString(DecimalStatus decSt, int scale, string& to) const; - - DecimalFixed abs() const; - DecimalFixed neg() const; - DecimalFixed add(DecimalStatus decSt, DecimalFixed op2) const; - DecimalFixed sub(DecimalStatus decSt, DecimalFixed op2) const; - DecimalFixed mul(DecimalStatus decSt, DecimalFixed op2) const; - DecimalFixed div(DecimalStatus decSt, DecimalFixed op2, int scale) const; - DecimalFixed mod(DecimalStatus decSt, DecimalFixed op2) const; - - DecimalFixed operator=(Decimal128Base d128b) - { - memcpy(&dec, &d128b.dec, sizeof(dec)); - return *this; - } - - void exactInt(DecimalStatus decSt, int scale); // rescale & make it integer after conversions - -private: - Decimal128 scaled128(DecimalStatus decSt, int scale) const; }; } // namespace Firebird diff --git a/src/common/Int128.cpp b/src/common/Int128.cpp new file mode 100644 index 0000000000..001e00534d --- /dev/null +++ b/src/common/Int128.cpp @@ -0,0 +1,492 @@ +/* + * PROGRAM: Integer 128 type. + * MODULE: Int128.cpp + * DESCRIPTION: Big integer support. + * + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Alex Peshkov + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2019 Alex Peshkov + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + * + * + */ + +#include "firebird.h" +#include "Int128.h" + +#include "StatusArg.h" +#include "gen/iberror.h" +#include "status.h" +#include "DecFloat.h" + +#include + +#include +#include +#include + +using namespace Firebird; + +namespace { + +const CInt128 i64max(MAX_SINT64), i64min(MIN_SINT64); +const double p2_32 = 4294967296.0; +const I128limit i128limit; +const CInt128 minus1(-1); + + +} // anonymous namespace + + + +namespace Firebird { + +Int128 Int128::set(SLONG value, int scale) +{ + v = value; + setScale(scale); + return *this; +} + +Int128 Int128::set(SINT64 value, int scale) +{ +#ifdef TTMATH_PLATFORM32 + v = ttmath::slint(value); +#else + v = ttmath::sint(value); +#endif + setScale(scale); + return *this; +} + +Int128 Int128::set(const char* value) +{ +// This is simplified method - it does not perform all what's needed for CVT_decompose + v.FromString(value); + return *this; +} + +Int128 Int128::set(double value) +{ + bool sgn = false; + if (value < 0.0) + { + value = -value; + sgn = true; + } + + double parts[4]; + for (int i = 0; i < 4; ++i) + { + parts[i] = value; + value /= p2_32; + } + fb_assert(value < 1.0); + + unsigned dwords[4]; + value = 0.0; + for (int i = 4; i--;) + { + dwords[i] = (parts[i] - value); + value += p2_32 * dwords[i]; + } + + setTable32(dwords); + if (sgn) + v.ChangeSign(); + + return *this; +} + +Int128 Int128::set(DecimalStatus decSt, Decimal128 value) +{ + static CDecimal128 quant(1); + value = value.quantize(decSt, quant); + + Decimal128::BCD bcd; + value.getBcd(&bcd); + fb_assert(bcd.exp == 0); + + v.SetZero(); + for (unsigned b = 0; b < sizeof(bcd.bcd); ++b) + { + v.MulInt(10); + v.AddInt(bcd.bcd[b]); + } + if (bcd.sign < 0) + v.ChangeSign(); + + return *this; +} + +void Int128::setScale(int scale) +{ + if (scale > 0) + { + ttmath::sint rem = 0; + while (scale--) + v.DivInt(10, scale == 0 ? &rem : nullptr); + + if (rem > 4) + v++; + else if (rem < -4) + v--; + } + else if (scale < 0) + { + while (scale++) { + if (v > i128limit.v || v < -i128limit.v) + (Arg::Gds(isc_arith_except) << Arg::Gds(isc_numeric_out_of_range)).raise(); + v.MulInt(10); + } + } +} + +int Int128::toInteger(int scale) const +{ + Int128 tmp(*this); + tmp.setScale(scale); + int rc; + if (tmp.v.ToInt(rc)) + overflow(); + return rc; +} + +void Int128::toString(int scale, unsigned length, char* to) const +{ + string buffer; + toString(scale, buffer); + if (buffer.length() + 1 > length) + { + (Arg::Gds(isc_arith_except) << Arg::Gds(isc_string_truncation) << + Arg::Gds(isc_trunc_limits) << Arg::Num(length) << Arg::Num(buffer.length() + 1)).raise(); + } + buffer.copyTo(to, length); +} + +void Int128::toString(int scale, string& to) const +{ + v.ToStringBase(to); + bool sgn = to[0] == '-'; + if (sgn) + to.erase(0, 1); + + if (scale) + { + if (scale < -38 || scale > 4) + { + string tmp; + tmp.printf("E%d", scale); + to += tmp; + } + else if (scale > 0) + { + string tmp(scale, '0'); + to += tmp; + } + else + { + unsigned posScale = -scale; + if (posScale > to.length()) + { + string tmp(posScale - to.length(), '0'); + to.insert(0, tmp); + } + if (posScale == to.length()) + { + to.insert(0, "0."); + } + else + to.insert(to.length() - posScale, "."); + } + } + + if (sgn) + to.insert(0, "-"); +} + +SINT64 Int128::toInt64(int scale) const +{ + Int128 tmp(*this); + tmp.setScale(scale); + if (tmp.v < i64min.v || tmp.v > i64max.v) + overflow(); + + unsigned dwords[4]; + tmp.getTable32(dwords); + SINT64 rc = int(dwords[1]); + rc <<= 32; + rc += dwords[0]; + + return rc; +} + +double Int128::toDouble() const +{ + unsigned dwords[4]; + getTable32(dwords); + double rc = int(dwords[3]); + for (int i = 3; i--;) + { + rc *= p2_32; + rc += dwords[i]; + } + + return rc; +} + +int Int128::compare(Int128 tgt) const +{ + return v < tgt.v ? -1 : v > tgt.v ? 1 : 0; +} + +Int128 Int128::abs() const +{ + Int128 rc(*this); + if (rc.v.Abs()) + overflow(); + return rc; +} + +Int128 Int128::neg() const +{ + Int128 rc(*this); + if (rc.v.ChangeSign()) + overflow(); + return rc; +} + +Int128 Int128::add(Int128 op2) const +{ + Int128 rc(*this); + if (rc.v.Add(op2.v)) + overflow(); + return rc; +} + +Int128 Int128::sub(Int128 op2) const +{ + Int128 rc(*this); + if (rc.v.Sub(op2.v)) + overflow(); + return rc; +} + +Int128 Int128::mul(Int128 op2) const +{ + Int128 rc(*this); + if (rc.v.Mul(op2.v)) + overflow(); + return rc; +} + +Int128 Int128::div(Int128 op2, int scale) const +{ + if (compare(MIN_Int128) == 0 && op2.compare(minus1) == 0) + Arg::Gds(isc_exception_integer_overflow).raise(); + + static const CInt128 MIN_BY10(MIN_Int128 / 10); + static const CInt128 MAX_BY10(MAX_Int128 / 10); + + // Scale op1 by as many of the needed powers of 10 as possible without an overflow. + CInt128 op1(*this); + int sign1 = op1.sign(); + while ((scale < 0) && (sign1 >= 0 ? op1.compare(MAX_BY10) <= 0 : op1.compare(MIN_BY10) >= 0)) + { + op1 *= 10; + ++scale; + } + + // Scale op2 shifting it to the right as long as only zeroes are thrown away. + CInt128 tmp(op2); + while (scale < 0) + { + ttmath::sint rem = 0; + tmp.v.DivInt(10, &rem); + if (rem) + break; + op2 = tmp; + ++scale; + } + + if (op1.v.Div(op2.v)) + zerodivide(); + + op1.setScale(scale); + return op1; +} + +Int128 Int128::mod(Int128 op2) const +{ + Int128 tmp(*this); + Int128 rc; + if (tmp.v.Div(op2.v, rc.v)) + zerodivide(); + return rc; +} + +int Int128::sign() const +{ + return v.IsSign() ? -1 : v.IsZero() ? 0 : 1; +} + +UCHAR* Int128::getBytes() +{ + return (UCHAR*)(v.table); +} + +void Int128::getTable32(unsigned* dwords) const +{ + static_assert((sizeof(v.table[0]) == 4) || (sizeof(v.table[0]) == 8), + "Unsupported size of integer in ttmath"); + + if (sizeof(v.table[0]) == 4) + { + for (int i = 0; i < 4; ++i) + dwords[i] = v.table[i]; + } + else if (sizeof(v.table[0]) == 8) + { + for (int i = 0; i < 2; ++i) + { + dwords[i * 2] = v.table[i] & 0xFFFFFFFF; + dwords[i * 2 + 1] = (v.table[i] >> 32) & 0xFFFFFFFF; + } + } +} + +void Int128::setTable32(const unsigned* dwords) +{ + static_assert((sizeof(v.table[0]) == 4) || (sizeof(v.table[0]) == 8), + "Unsupported size of integer in ttmath"); + + if (sizeof(v.table[0]) == 4) + { + for (int i = 0; i < 4; ++i) + v.table[i] = dwords[i]; + } + else if (sizeof(v.table[0]) == 8) + { + for (int i = 0; i < 2; ++i) + { + v.table[i] = dwords[i * 2 + 1]; + v.table[i] <<= 32; + v.table[i] += dwords[i * 2]; + } + } +} + +Int128 Int128::operator&=(FB_UINT64 mask) +{ + v.table[0] &= mask; + unsigned i = 1; + if (sizeof(v.table[0]) == 4) + { + i = 2; + v.table[1] &= (mask >> 32); + } + + for (; i < FB_NELEM(v.table); ++i) + v.table[i] = 0; + return *this; +} + +Int128 Int128::operator&=(ULONG mask) +{ + v.table[0] &= mask; + + for (unsigned i = 1; i < FB_NELEM(v.table); ++i) + v.table[i] = 0; + return *this; +} + +Int128 Int128::operator/(unsigned value) const +{ + Int128 rc; + rc.v.DivInt(value); + return rc; +} + +Int128 Int128::operator-() const +{ + return neg(); +} + +Int128 Int128::operator+=(unsigned value) +{ + v.AddInt(value); + return *this; +} + +Int128 Int128::operator*=(unsigned value) +{ + v.MulInt(value); + return *this; +} + +bool Int128::operator>(Int128 value) const +{ + return v > value.v; +} + +bool Int128::operator==(Int128 value) const +{ + return v == value.v; +} + +void Int128::zerodivide() +{ + (Arg::Gds(isc_arith_except) << Arg::Gds(isc_exception_integer_divide_by_zero)).raise(); +} + +void Int128::overflow() +{ + (Arg::Gds(isc_arith_except) << Arg::Gds(isc_exception_integer_overflow)).raise(); +} + +#ifdef DEV_BUILD +const char* Int128::show() +{ + static char to[64]; + toString(0, sizeof(to), to); + return to; +} +#endif + +CInt128::CInt128(SINT64 value) +{ + set(value, 0); +} + +CInt128::CInt128(minmax mm) +{ + switch(mm) + { + case MkMax: + v.SetMax(); + break; + case MkMin: + v.SetMin(); + break; + } +} + +CInt128 MIN_Int128(CInt128::MkMin); +CInt128 MAX_Int128(CInt128::MkMax); + +} // namespace Firebird diff --git a/src/common/Int128.h b/src/common/Int128.h new file mode 100644 index 0000000000..758e387174 --- /dev/null +++ b/src/common/Int128.h @@ -0,0 +1,147 @@ +/* + * PROGRAM: Integer 128 type. + * MODULE: Int128.h + * DESCRIPTION: Big integer support. + * + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Alex Peshkov + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2019 Alex Peshkov + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + * + * + */ + +#ifndef FB_INT128 +#define FB_INT128 + +#include "firebird/Interface.h" +#include "fb_exception.h" + +#include + +#include "classes/fb_string.h" +#include "classes/MetaName.h" + +#include "../../extern/ttmath/ttmath.h" + +namespace Firebird { + +class Decimal64; +class Decimal128; +struct DecimalStatus; + +class Int128 //: public Decimal128Base +{ +public: +#if SIZEOF_LONG < 8 + Int128 set(int value) + { + return set(SLONG(value)); + } +#endif + Int128 set(SLONG value, int scale); + Int128 set(SINT64 value, int scale); + Int128 set(double value); + Int128 set(DecimalStatus decSt, Decimal128 value); + Int128 set(Int128 value) + { + v = value.v; + return *this; + } + + Int128 operator=(SINT64 value) + { + set(value, 0); + return *this; + } + +#ifdef DEV_BUILD + const char* show(); +#endif + + int toInteger(int scale) const; + SINT64 toInt64(int scale) const; + void toString(int scale, unsigned length, char* to) const; + void toString(int scale, string& to) const; + double toDouble() const; + + Int128 operator&=(FB_UINT64 mask); + Int128 operator&=(ULONG mask); + Int128 operator-() const; + Int128 operator/(unsigned value) const; + Int128 operator+=(unsigned value); + Int128 operator*=(unsigned value); + + int compare(Int128 tgt) const; + bool operator>(Int128 value) const; + bool operator==(Int128 value) const; + int sign() const; + + Int128 abs() const; + Int128 neg() const; + Int128 add(Int128 op2) const; + Int128 sub(Int128 op2) const; + Int128 mul(Int128 op2) const; + Int128 div(Int128 op2, int scale) const; + Int128 mod(Int128 op2) const; + + void getTable32(unsigned* dwords) const; // internal data in per-32bit form + void setTable32(const unsigned* dwords); + void setScale(int scale); + UCHAR* getBytes(); + +protected: + ttmath::Int v; + + static void overflow(); + static void zerodivide(); + + Int128 set(const char* value); +}; + +class CInt128 : public Int128 +{ +public: + enum minmax {MkMax, MkMin}; + + CInt128(SINT64 value); + CInt128(minmax mm); + CInt128(const Int128& value) + { + set(value); + } +}; + +extern CInt128 MAX_Int128, MIN_Int128; + +class I128limit : public Int128 +{ +public: + I128limit() + { + v.SetOne(); + for (int i = 0; i < 126; ++i) + v.MulInt(2); + v.DivInt(5); + } +}; + +} // namespace Firebird + + +#endif // FB_INT128 diff --git a/src/common/classes/InternalMessageBuffer.cpp b/src/common/classes/InternalMessageBuffer.cpp index fbd153763d..45c5a44a86 100644 --- a/src/common/classes/InternalMessageBuffer.cpp +++ b/src/common/classes/InternalMessageBuffer.cpp @@ -192,9 +192,9 @@ MetadataFromBlr::MetadataFromBlr(unsigned aBlrLength, const unsigned char* aBlr, item->length = sizeof(Decimal128); break; - case blr_dec_fixed: - item->type = SQL_DEC_FIXED; - item->length = sizeof(DecimalFixed); + case blr_int128: + item->type = SQL_INT128; + item->length = sizeof(Int128); item->scale = rdr.getByte(); break; diff --git a/src/common/classes/fb_string.h b/src/common/classes/fb_string.h index f9d6c1b5ca..b759c1f82a 100644 --- a/src/common/classes/fb_string.h +++ b/src/common/classes/fb_string.h @@ -504,7 +504,10 @@ namespace Firebird memset(baseInsert(p0, n), c, n); return *this; } - // iterator insert(iterator it, char_type c); // what to return here? + void insert(iterator it, char_type c) + { + insert(it - c_str(), 1, c); + } void insert(iterator it, size_type n, char_type c) { insert(it - c_str(), n, c); @@ -519,6 +522,11 @@ namespace Firebird baseErase(p0, n); return *this; } + AbstractString& clear() throw() + { + erase(); + return *this; + } iterator erase(iterator it) throw() { erase(it - c_str(), 1); diff --git a/src/common/cvt.cpp b/src/common/cvt.cpp index 76f7acca82..debc9cff6f 100644 --- a/src/common/cvt.cpp +++ b/src/common/cvt.cpp @@ -133,7 +133,7 @@ static void datetime_to_text(const dsc*, dsc*, Callbacks*); static void float_to_text(const dsc*, dsc*, Callbacks*); static void decimal_float_to_text(const dsc*, dsc*, DecimalStatus, Callbacks*); static void integer_to_text(const dsc*, dsc*, Callbacks*); -static SINT64 hex_to_value(const char*& string, const char* end); +static void int128_to_text(const dsc*, dsc*, Callbacks* cb); static void localError(const Firebird::Arg::StatusVector&); class DummyException {}; @@ -305,7 +305,35 @@ static void decimal_float_to_text(const dsc* from, dsc* to, DecimalStatus decSt, else if (from->dsc_dtype == dtype_dec128) ((Decimal128*) from->dsc_address)->toString(decSt, sizeof(temp), temp); else - ((DecimalFixed*) from->dsc_address)->toString(decSt, from->dsc_scale, sizeof(temp), temp); + fb_assert(false); + } + catch (const Exception& ex) + { + // reraise using function passed in callbacks + Arg::StatusVector v(ex); + cb->err(v); + } + + dsc intermediate; + intermediate.dsc_dtype = dtype_text; + intermediate.dsc_ttype() = ttype_ascii; + intermediate.dsc_address = reinterpret_cast(temp); + intermediate.dsc_length = strlen(temp); + + CVT_move_common(&intermediate, to, 0, cb); +} + + +static void int128_to_text(const dsc* from, dsc* to, Callbacks* cb) +{ + char temp[50]; + + try + { + if (from->dsc_dtype == dtype_int128) + ((Int128*) from->dsc_address)->toString(from->dsc_scale, sizeof(temp), temp); + else + fb_assert(false); } catch (const Exception& ex) { @@ -1070,9 +1098,8 @@ SLONG CVT_get_long(const dsc* desc, SSHORT scale, DecimalStatus decSt, ErrorFunc return d128.toInteger(decSt, scale); - case dtype_dec_fixed: - value = ((DecimalFixed*) p)->toInteger(decSt); - break; + case dtype_int128: + return ((Int128*) p)->toInteger(scale); case dtype_real: case dtype_double: @@ -1121,7 +1148,7 @@ SLONG CVT_get_long(const dsc* desc, SSHORT scale, DecimalStatus decSt, ErrorFunc { USHORT length = CVT_make_string(desc, ttype_ascii, &p, &buffer, sizeof(buffer), decSt, err); - scale -= CVT_decompose(p, length, dtype_long, &value, err); + scale -= CVT_decompose(p, length, &value, err); } break; @@ -1274,8 +1301,8 @@ double CVT_get_double(const dsc* desc, DecimalStatus decSt, ErrorFunction err, b return d128.toDouble(decSt); } - case dtype_dec_fixed: - value = ((DecimalFixed*) desc->dsc_address)->toDouble(decSt); + case dtype_int128: + value = ((Int128*) desc->dsc_address)->toDouble(); break; case dtype_varying: @@ -1578,7 +1605,7 @@ void CVT_move_common(const dsc* from, dsc* to, DecimalStatus decSt, Callbacks* c case dtype_boolean: case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: CVT_conversion_error(from, cb->err); break; } @@ -1629,7 +1656,7 @@ void CVT_move_common(const dsc* from, dsc* to, DecimalStatus decSt, Callbacks* c case dtype_boolean: case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: CVT_conversion_error(from, cb->err); break; } @@ -1671,7 +1698,7 @@ void CVT_move_common(const dsc* from, dsc* to, DecimalStatus decSt, Callbacks* c case dtype_boolean: case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: CVT_conversion_error(from, cb->err); break; } @@ -1716,7 +1743,7 @@ void CVT_move_common(const dsc* from, dsc* to, DecimalStatus decSt, Callbacks* c case dtype_boolean: case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: CVT_conversion_error(from, cb->err); break; } @@ -1763,7 +1790,7 @@ void CVT_move_common(const dsc* from, dsc* to, DecimalStatus decSt, Callbacks* c case dtype_boolean: case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: CVT_conversion_error(from, cb->err); break; } @@ -1946,6 +1973,10 @@ void CVT_move_common(const dsc* from, dsc* to, DecimalStatus decSt, Callbacks* c integer_to_text(from, to, cb); return; + case dtype_int128: + int128_to_text(from, to, cb); + return; + case dtype_real: case dtype_double: float_to_text(from, to, cb); @@ -1953,7 +1984,6 @@ void CVT_move_common(const dsc* from, dsc* to, DecimalStatus decSt, Callbacks* c case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: decimal_float_to_text(from, to, decSt, cb); return; @@ -2082,8 +2112,8 @@ void CVT_move_common(const dsc* from, dsc* to, DecimalStatus decSt, Callbacks* c *((Decimal128*) p) = CVT_get_dec128(from, decSt, cb->err); return; - case dtype_dec_fixed: - *((DecimalFixed*) p) = CVT_get_dec_fixed(from, (SSHORT) to->dsc_scale, decSt, cb->err); + case dtype_int128: + *((Int128*) p) = CVT_get_int128(from, (SSHORT) to->dsc_scale, decSt, cb->err); return; case dtype_boolean: @@ -2108,7 +2138,7 @@ void CVT_move_common(const dsc* from, dsc* to, DecimalStatus decSt, Callbacks* c case dtype_double: case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: CVT_conversion_error(from, cb->err); break; } @@ -2494,11 +2524,28 @@ double CVT_power_of_ten(const int scale) } -SSHORT CVT_decompose(const char* string, - USHORT length, - SSHORT dtype, - SLONG* return_value, - ErrorFunction err) +class RetPtr +{ +public: + virtual ~RetPtr() { } + + enum lb10 {RETVAL_OVERFLOW, RETVAL_POSSIBLE_OVERFLOW, RETVAL_NO_OVERFLOW}; + + virtual USHORT maxSize() = 0; + virtual void truncate8() = 0; + virtual void truncate16() = 0; + virtual lb10 compareLimitBy10() = 0; + virtual void nextDigit(unsigned digit, unsigned base) = 0; + virtual bool isLowerLimit() = 0; + virtual void neg() = 0; +}; + +static void hex_to_value(const char*& string, const char* end, RetPtr* retValue); + +static SSHORT cvt_decompose(const char* string, + USHORT length, + RetPtr* return_value, + ErrorFunction err) { /************************************** * @@ -2512,15 +2559,6 @@ SSHORT CVT_decompose(const char* string, * **************************************/ - // For now, this routine does not handle quadwords unless this is - // supported by the platform as a native datatype. - - if (dtype == dtype_quad) - { - fb_assert(false); - err(Arg::Gds(isc_badblk)); // internal error - } - dsc errd; MOVE_CLEAR(&errd, sizeof(errd)); errd.dsc_dtype = dtype_text; @@ -2528,14 +2566,9 @@ SSHORT CVT_decompose(const char* string, errd.dsc_length = length; errd.dsc_address = reinterpret_cast(const_cast(string)); - SINT64 value = 0; SSHORT scale = 0; int sign = 0; bool digit_seen = false, fraction = false; - const SINT64 lower_limit = (dtype == dtype_long) ? MIN_SLONG : MIN_SINT64; - const SINT64 upper_limit = (dtype == dtype_long) ? MAX_SLONG : MAX_SINT64; - - const SINT64 limit_by_10 = upper_limit / 10; // used to check for overflow const char* p = string; const char* end = p + length; @@ -2561,28 +2594,20 @@ SSHORT CVT_decompose(const char* string, while (q < end && *q == ' ') q++; - if (q != end || end - p == 0 || end - p > 16) + if (q != end || end - p == 0 || end - p > return_value->maxSize()) CVT_conversion_error(&errd, err); q = p; - value = hex_to_value(q, digits_end); + hex_to_value(q, digits_end, return_value); if (q != digits_end) CVT_conversion_error(&errd, err); // 0xFFFFFFFF = -1; 0x0FFFFFFFF = 4294967295 if (digits_end - p <= 8) - value = (SLONG) value; - - if (dtype == dtype_long) - { - if (value < LONG_MIN_int64 || value > LONG_MAX_int64) - err(Arg::Gds(isc_arith_except) << Arg::Gds(isc_numeric_out_of_range)); - - *return_value = (SLONG) value; - } - else - *((SINT64*) return_value) = value; + return_value->truncate8(); + else if (digits_end - p <= 16) + return_value->truncate16(); return 0; // 0 scale for hex literals } @@ -2598,20 +2623,33 @@ SSHORT CVT_decompose(const char* string, // tricky: the value doesn't always become negative after an // overflow! - if (value >= limit_by_10) + switch(return_value->compareLimitBy10()) { - // possibility of an overflow - if (value > limit_by_10) - { - err(Arg::Gds(isc_arith_except) << Arg::Gds(isc_numeric_out_of_range)); - } - else if ((*p > '8' && sign == -1) || (*p > '7' && sign != -1)) + case RetPtr::RETVAL_OVERFLOW: + if (fraction) + { + while (p < end) + { + if (*p != '0') + break; + ++p; + } + if (p >= end) + continue; + } + err(Arg::Gds(isc_arith_except) << Arg::Gds(isc_numeric_out_of_range)); + break; + case RetPtr::RETVAL_POSSIBLE_OVERFLOW: + if ((*p > '8' && sign == -1) || (*p > '7' && sign != -1)) { err(Arg::Gds(isc_arith_except) << Arg::Gds(isc_numeric_out_of_range)); } + break; + default: + break; } - value = value * 10 + *p - '0'; + return_value->nextDigit(*p - '0', 10); if (fraction) --scale; } @@ -2645,8 +2683,8 @@ SSHORT CVT_decompose(const char* string, if (!digit_seen) CVT_conversion_error(&errd, err); - if ((sign == -1) && value != lower_limit) - value = -value; + if ((sign == -1) && !return_value->isLowerLimit()) + return_value->neg(); // If there's still something left, there must be an explicit exponent if (p < end) @@ -2693,18 +2731,160 @@ SSHORT CVT_decompose(const char* string, if (!digit_seen) CVT_conversion_error(&errd, err); - } - if (dtype == dtype_long) - *return_value = (SLONG) value; - else - *((SINT64 *) return_value) = value; - return scale; } +template +class RetValue : public RetPtr +{ +public: + RetValue(typename Traits::ValueType* ptr) + : return_value(ptr) + { + value = 0; + } + + ~RetValue() + { + *return_value = value; + } + + USHORT maxSize() + { + return sizeof(typename Traits::ValueType); + } + + void truncate8() + { + ULONG mask = 0xFFFFFFFF; + value &= mask; + } + + void truncate16() + { + FB_UINT64 mask = 0xFFFFFFFFFFFFFFFF; + value &= mask; + } + + lb10 compareLimitBy10() + { + if (value > Traits::UPPER_LIMIT / 10) + return RETVAL_OVERFLOW; + if (value == Traits::UPPER_LIMIT / 10) + return RETVAL_POSSIBLE_OVERFLOW; + return RETVAL_NO_OVERFLOW; + } + + void nextDigit(unsigned digit, unsigned base) + { + value *= base; + value += digit; + } + + bool isLowerLimit() + { + return value == Traits::LOWER_LIMIT; + } + + void neg() + { + value = -value; + } + +private: + typename Traits::ValueType value; + typename Traits::ValueType* return_value; +}; + + +class SLONGTraits +{ +public: + typedef SLONG ValueType; + static const SLONG UPPER_LIMIT = MAX_SLONG; + static const SLONG LOWER_LIMIT = MIN_SLONG; +}; + +SSHORT CVT_decompose(const char* str, USHORT len, SLONG* val, ErrorFunction err) +{ +/************************************** + * + * d e c o m p o s e + * + ************************************** + * + * Functional description + * Decompose a numeric string in mantissa and exponent, + * or if it is in hexadecimal notation. + * + **************************************/ + + RetValue value(val); + return cvt_decompose(str, len, &value, err); +} + + +class SINT64Traits +{ +public: + typedef SINT64 ValueType; + static const SINT64 UPPER_LIMIT = MAX_SINT64; + static const SINT64 LOWER_LIMIT = MIN_SINT64; +}; + +SSHORT CVT_decompose(const char* str, USHORT len, SINT64* val, ErrorFunction err) +{ +/************************************** + * + * d e c o m p o s e + * + ************************************** + * + * Functional description + * Decompose a numeric string in mantissa and exponent, + * or if it is in hexadecimal notation. + * + **************************************/ + + RetValue value(val); + return cvt_decompose(str, len, &value, err); +} + + +class I128Traits +{ +public: + typedef Int128 ValueType; + static const CInt128 UPPER_LIMIT; + static const CInt128 LOWER_LIMIT; +}; + +const CInt128 I128Traits::UPPER_LIMIT(MAX_Int128); +const CInt128 I128Traits::LOWER_LIMIT(MIN_Int128); + +SSHORT CVT_decompose(const char* str, USHORT len, Int128* val, ErrorFunction err) +{ +/************************************** + * + * d e c o m p o s e + * + ************************************** + * + * Functional description + * Decompose a numeric string in mantissa and exponent, + * or if it is in hexadecimal notation. + * + **************************************/ + + + RetValue value(val); + return cvt_decompose(str, len, &value, err); +} + + USHORT CVT_get_string_ptr_common(const dsc* desc, USHORT* ttype, UCHAR** address, vary* temp, USHORT length, DecimalStatus decSt, Callbacks* cb) { @@ -2851,10 +3031,10 @@ Decimal64 CVT_get_dec64(const dsc* desc, DecimalStatus decSt, ErrorFunction err) return *(Decimal64*) p; case dtype_dec128: - return ((Decimal128Base*) p)->toDecimal64(decSt); + return ((Decimal128*) p)->toDecimal64(decSt); - case dtype_dec_fixed: - return d64.set(*((DecimalFixed*) p), decSt, scale); + case dtype_int128: + return d64.set(*((Int128*) p), decSt, scale); default: fb_assert(false); @@ -2940,8 +3120,8 @@ Decimal128 CVT_get_dec128(const dsc* desc, DecimalStatus decSt, ErrorFunction er case dtype_dec128: return *(Decimal128*) p; - case dtype_dec_fixed: - return d128.set(*((DecimalFixed*) p), decSt, scale); + case dtype_int128: + return d128.set(*((Int128*) p), decSt, scale); default: fb_assert(false); @@ -2961,7 +3141,7 @@ Decimal128 CVT_get_dec128(const dsc* desc, DecimalStatus decSt, ErrorFunction er } -DecimalFixed CVT_get_dec_fixed(const dsc* desc, SSHORT scale, DecimalStatus decSt, ErrorFunction err) +Int128 CVT_get_int128(const dsc* desc, SSHORT scale, DecimalStatus decSt, ErrorFunction err) { /************************************** * @@ -2974,8 +3154,15 @@ DecimalFixed CVT_get_dec_fixed(const dsc* desc, SSHORT scale, DecimalStatus decS * **************************************/ VaryStr<1024> buffer; // represents unreasonably long decfloat literal in ASCII - DecimalFixed dfix; + Int128 int128; Decimal128 tmp; + double d, eps; + + static const double I128_MIN_dbl = -1.701411834604692e+38; + static const double I128_MAX_dbl = 1.701411834604692e+38; + static const CDecimal128 I128_MIN_dcft("-1.701411834604692317316873037158841E+38", decSt); + static const CDecimal128 I128_MAX_dcft("1.701411834604692317316873037158841E+38", decSt); + static const CDecimal128 DecFlt_05("0.5", decSt); // adjust exact numeric values to same scaling if (DTYPE_IS_EXACT(desc->dsc_dtype)) @@ -2988,27 +3175,30 @@ DecimalFixed CVT_get_dec_fixed(const dsc* desc, SSHORT scale, DecimalStatus decS switch (desc->dsc_dtype) { case dtype_short: - dfix.set(*(SSHORT*) p); + int128.set(SLONG(*(SSHORT*) p), scale); break; case dtype_long: - dfix.set(*(SLONG*) p); + int128.set(*(SLONG*) p, scale); break; case dtype_quad: - dfix.set(CVT_get_int64(desc, 0, decSt, err)); + int128.set(CVT_get_int64(desc, 0, decSt, err), scale); break; case dtype_int64: - dfix.set(*(SINT64*) p); + int128.set(*(SINT64*) p, scale); break; case dtype_varying: case dtype_cstring: case dtype_text: - CVT_make_null_string(desc, ttype_ascii, &p, &buffer, sizeof(buffer) - 1, decSt, err); - dfix.set(buffer.vary_string, scale, decSt); - return dfix; // scale already corrected + { + USHORT length = + CVT_make_string(desc, ttype_ascii, &p, &buffer, sizeof(buffer), decSt, err); + scale -= CVT_decompose(p, length, &int128, err); + int128.setScale(scale); + } break; case dtype_blob: @@ -3022,23 +3212,70 @@ DecimalFixed CVT_get_dec_fixed(const dsc* desc, SSHORT scale, DecimalStatus decS break; case dtype_real: - dfix.set(*((float*) p), scale, decSt); - return dfix; // scale already corrected - case dtype_double: - dfix.set(*((double*) p), scale, decSt); - return dfix; // scale already corrected + if (desc->dsc_dtype == dtype_real) + { + d = *((float*) p); + eps = eps_float; + } + else // if (desc->dsc_dtype == DEFAULT_DOUBLE) + { + d = *((double*) p); + eps = eps_double; + } + + if (scale > 0) + d /= CVT_power_of_ten(scale); + else if (scale < 0) + d *= CVT_power_of_ten(-scale); + + if (d > 0) + d += 0.5 + eps; + else + d -= 0.5 + eps; + + /* make sure the cast will succeed + + Note that adding or subtracting 0.5, as we do in CVT_get_long, + will never allow the rounded value to fit into an Int128, + because when the double value is too large in magnitude + to fit, 0.5 is less than half of the least significant bit + of the significant (sometimes miscalled "mantissa") of the + double, and thus will have no effect on the sum. */ + + if (d < I128_MIN_dbl || I128_MAX_dbl < d) + err(Arg::Gds(isc_arith_except) << Arg::Gds(isc_numeric_out_of_range)); + + int128.set(d); + break; case dtype_dec64: - dfix = tmp = *((Decimal64*) p); - break; - case dtype_dec128: - dfix = *((Decimal128*) p); + if (desc->dsc_dtype == dtype_dec64) + tmp = *((Decimal64*) p); + else + tmp = *((Decimal128*) p); + + tmp.setScale(decSt, -scale); + + /* make sure the cast will succeed + + Note that adding or subtracting 0.5, as we do in CVT_get_long, + will never allow the rounded value to fit into an Int128, + because when the double value is too large in magnitude + to fit, 0.5 is less than half of the least significant bit + of the significant (sometimes miscalled "mantissa") of the + double, and thus will have no effect on the sum. */ + + if (tmp.compare(decSt, I128_MIN_dcft) < 0 || I128_MAX_dcft.compare(decSt, tmp) < 0) + err(Arg::Gds(isc_arith_except) << Arg::Gds(isc_numeric_out_of_range)); + + int128.set(decSt, tmp); break; - case dtype_dec_fixed: - dfix = *((DecimalFixed*) p); + case dtype_int128: + int128 = *((Int128*) p); + int128.setScale(scale); break; default: @@ -3054,8 +3291,7 @@ DecimalFixed CVT_get_dec_fixed(const dsc* desc, SSHORT scale, DecimalStatus decS err(v); } - dfix.exactInt(decSt, scale); - return dfix; + return int128; } @@ -3154,7 +3390,9 @@ SQUAD CVT_get_quad(const dsc* desc, SSHORT scale, DecimalStatus decSt, ErrorFunc { USHORT length = CVT_make_string(desc, ttype_ascii, &p, &buffer, sizeof(buffer), decSt, err); - scale -= CVT_decompose(p, length, dtype_quad, &value.gds_quad_high, err); + SINT64 i64; + scale -= CVT_decompose(p, length, &i64, err); + SINT64_to_SQUAD(i64, value); } break; @@ -3170,7 +3408,7 @@ SQUAD CVT_get_quad(const dsc* desc, SSHORT scale, DecimalStatus decSt, ErrorFunc case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: SINT64_to_SQUAD(CVT_get_int64(desc, scale, decSt, err), value); break; @@ -3246,9 +3484,8 @@ SINT64 CVT_get_int64(const dsc* desc, SSHORT scale, DecimalStatus decSt, ErrorFu return d128.toInt64(decSt, scale); } - case dtype_dec_fixed: - value = ((DecimalFixed*) p)->toInt64(decSt); - break; + case dtype_int128: + return ((Int128*) p)->toInt64(scale); case dtype_real: case dtype_double: @@ -3295,7 +3532,7 @@ SINT64 CVT_get_int64(const dsc* desc, SSHORT scale, DecimalStatus decSt, ErrorFu { USHORT length = CVT_make_string(desc, ttype_ascii, &p, &buffer, sizeof(buffer), decSt, err); - scale -= CVT_decompose(p, length, dtype_int64, (SLONG *) & value, err); + scale -= CVT_decompose(p, length, &value, err); } break; @@ -3349,7 +3586,7 @@ SINT64 CVT_get_int64(const dsc* desc, SSHORT scale, DecimalStatus decSt, ErrorFu } -static SINT64 hex_to_value(const char*& string, const char* end) +static void hex_to_value(const char*& string, const char* end, RetPtr* retValue) /************************************* * * hex_to_value @@ -3366,7 +3603,6 @@ static SINT64 hex_to_value(const char*& string, const char* end) // we already know this is a hex string, and there is no prefix. // So, string is something like DEADBEEF. - SINT64 value = 0; UCHAR byte = 0; int nibble = ((end - string) & 1); char ch; @@ -3385,7 +3621,7 @@ static SINT64 hex_to_value(const char*& string, const char* end) { byte = (byte << 4) + (UCHAR) c; nibble = 0; - value = (value << 8) + byte; + retValue->nextDigit(byte, 256); } else { @@ -3397,8 +3633,6 @@ static SINT64 hex_to_value(const char*& string, const char* end) } fb_assert(string <= end); - - return value; } diff --git a/src/common/cvt.h b/src/common/cvt.h index b7e5dd31f7..5359fb6182 100644 --- a/src/common/cvt.h +++ b/src/common/cvt.h @@ -80,7 +80,9 @@ enum EXPECT_DATETIME expect_sql_time_tz }; -} +class Int128; + +} // namespace Firebird void CVT_conversion_error(const dsc*, ErrorFunction); @@ -90,12 +92,14 @@ bool CVT_get_boolean(const dsc*, ErrorFunction); double CVT_get_double(const dsc*, Firebird::DecimalStatus, ErrorFunction, bool* getNumericOverflow = nullptr); Firebird::Decimal64 CVT_get_dec64(const dsc*, Firebird::DecimalStatus, ErrorFunction); Firebird::Decimal128 CVT_get_dec128(const dsc*, Firebird::DecimalStatus, ErrorFunction); -Firebird::DecimalFixed CVT_get_dec_fixed(const dsc*, SSHORT, Firebird::DecimalStatus, ErrorFunction); +Firebird::Int128 CVT_get_int128(const dsc*, SSHORT, Firebird::DecimalStatus, ErrorFunction); USHORT CVT_make_string(const dsc*, USHORT, const char**, vary*, USHORT, Firebird::DecimalStatus, ErrorFunction); void CVT_make_null_string(const dsc*, USHORT, const char**, vary*, USHORT, Firebird::DecimalStatus, ErrorFunction); void CVT_move_common(const dsc*, dsc*, Firebird::DecimalStatus, Firebird::Callbacks*); void CVT_move(const dsc*, dsc*, Firebird::DecimalStatus, ErrorFunction); -SSHORT CVT_decompose(const char*, USHORT, SSHORT, SLONG*, ErrorFunction); +SSHORT CVT_decompose(const char*, USHORT, SLONG*, ErrorFunction); +SSHORT CVT_decompose(const char*, USHORT, SINT64*, ErrorFunction); +SSHORT CVT_decompose(const char*, USHORT, Firebird::Int128*, ErrorFunction); USHORT CVT_get_string_ptr(const dsc*, USHORT*, UCHAR**, vary*, USHORT, Firebird::DecimalStatus, ErrorFunction); USHORT CVT_get_string_ptr_common(const dsc*, USHORT*, UCHAR**, vary*, USHORT, Firebird::DecimalStatus, Firebird::Callbacks*); SINT64 CVT_get_int64(const dsc*, SSHORT, Firebird::DecimalStatus, ErrorFunction); diff --git a/src/common/dsc.cpp b/src/common/dsc.cpp index 75810a1729..6601a26b36 100644 --- a/src/common/dsc.cpp +++ b/src/common/dsc.cpp @@ -68,7 +68,7 @@ static const USHORT _DSC_convert_to_text_length[DTYPE_TYPE_MAX] = 5, // dtype_boolean 23, // dtype_dec64 1 + 1 + 1 + 1 + 16(34) + 3(4) 42, // dtype_dec128 +- . e +- coeff + exp - 36, // dtype_dec_fixed coeff(34) + 1(+-) + 1(.) + 47, // dtype_int128 14 + TimeZoneUtil::MAX_LEN, // dtype_sql_time_tz HH:MM:SS.MMMM +NN:NN 25 + TimeZoneUtil::MAX_LEN // dtype_timestamp_tz YYYY-MM-DD HH:MM:SS.MMMM +NN:NN }; @@ -122,7 +122,7 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double dtype_d_float dtype_sql_date dtype_sql_time dtype_timestamp dtype_blob dtype_array dtype_int64 dtype_dbkey dtype_boolean dtype_dec64 dtype_dec128 - dtype_dec_fixed dtype_sql_time_tz dtype_timestamp_tz + dtype_int128 dtype_sql_time_tz dtype_timestamp_tz */ // dtype_unknown @@ -141,7 +141,7 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, dtype_timestamp_tz}, + dtype_int128, DTYPE_CANNOT, dtype_timestamp_tz}, // dtype_cstring {dtype_unknown, dtype_double, dtype_double, dtype_double, @@ -150,7 +150,7 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, dtype_timestamp_tz}, + dtype_int128, DTYPE_CANNOT, dtype_timestamp_tz}, // dtype_varying {dtype_unknown, dtype_double, dtype_double, dtype_double, @@ -159,7 +159,7 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, dtype_timestamp_tz}, + dtype_int128, DTYPE_CANNOT, dtype_timestamp_tz}, // 4 (unused) {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -204,7 +204,7 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, dtype_sql_date, dtype_sql_time, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, dtype_sql_time_tz, dtype_timestamp_tz}, + dtype_int128, dtype_sql_time_tz, dtype_timestamp_tz}, // dtype_long {dtype_unknown, dtype_double, dtype_double, dtype_double, @@ -213,7 +213,7 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, dtype_sql_date, dtype_sql_time, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, dtype_sql_time_tz, dtype_timestamp_tz}, + dtype_int128, dtype_sql_time_tz, dtype_timestamp_tz}, // dtype_quad {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -303,7 +303,7 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, dtype_sql_date, dtype_sql_time, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, dtype_sql_time_tz, dtype_timestamp_tz}, + dtype_int128, dtype_sql_time_tz, dtype_timestamp_tz}, // dtype_dbkey {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -341,14 +341,14 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, dtype_dec128, dtype_sql_time_tz, dtype_timestamp_tz}, - // dtype_dec_fixed - {dtype_unknown, dtype_dec_fixed, dtype_dec_fixed, dtype_dec_fixed, + // dtype_int128 + {dtype_unknown, dtype_int128, dtype_int128, dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, - dtype_dec_fixed, dtype_dec_fixed, DTYPE_CANNOT, dtype_double, + dtype_int128, dtype_int128, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, dtype_sql_date, dtype_sql_time, - dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec_fixed, + dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, dtype_sql_time_tz, dtype_timestamp_tz}, + dtype_int128, dtype_sql_time_tz, dtype_timestamp_tz}, // dtype_sql_time_tz {dtype_unknown, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -384,7 +384,7 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double dtype_d_float dtype_sql_date dtype_sql_time dtype_timestamp dtype_blob dtype_array dtype_int64 dtype_dbkey dtype_boolean dtype_dec64 dtype_dec128 - dtype_dec_fixed dtype_sql_time_tz dtype_timestamp_tz + dtype_int128 dtype_sql_time_tz dtype_timestamp_tz */ // dtype_unknown @@ -403,7 +403,7 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, dtype_timestamp_tz}, + dtype_int128, DTYPE_CANNOT, dtype_timestamp_tz}, // dtype_cstring {dtype_unknown, dtype_double, dtype_double, dtype_double, @@ -412,7 +412,7 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, dtype_timestamp_tz}, + dtype_int128, DTYPE_CANNOT, dtype_timestamp_tz}, // dtype_varying {dtype_unknown, dtype_double, dtype_double, dtype_double, @@ -421,7 +421,7 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, dtype_timestamp_tz}, + dtype_int128, DTYPE_CANNOT, dtype_timestamp_tz}, // 4 (unused) {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -466,7 +466,7 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_long {dtype_unknown, dtype_double, dtype_double, dtype_double, @@ -475,7 +475,7 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_quad {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -565,7 +565,7 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_dbkey {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -603,14 +603,14 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT}, - // dtype_dec_fixed - {dtype_unknown, dtype_dec_fixed, dtype_dec_fixed, dtype_dec_fixed, + // dtype_int128 + {dtype_unknown, dtype_int128, dtype_int128, dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, - dtype_dec_fixed, dtype_dec_fixed, DTYPE_CANNOT, dtype_double, + dtype_int128, dtype_int128, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, - DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec_fixed, + DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_sql_time_tz {dtype_unknown, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -647,7 +647,7 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double dtype_d_float dtype_sql_date dtype_sql_time dtype_timestamp dtype_blob dtype_array dtype_int64 dtype_dbkey dtype_boolean dtype_dec64 dtype_dec128 - dtype_dec_fixed dtype_sql_time_tz dtype_timestamp_tz + dtype_int128 dtype_sql_time_tz dtype_timestamp_tz */ // dtype_unknown @@ -666,7 +666,7 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_cstring {dtype_unknown, dtype_double, dtype_double, dtype_double, @@ -675,7 +675,7 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_varying {dtype_unknown, dtype_double, dtype_double, dtype_double, @@ -684,7 +684,7 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // 4 (unused) {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -729,7 +729,7 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_long {dtype_unknown, dtype_double, dtype_double, dtype_double, @@ -738,7 +738,7 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_quad {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -826,9 +826,9 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, dtype_int64, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, - DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, + DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_dbkey {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -866,14 +866,14 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT}, - // dtype_dec_fixed - {dtype_unknown, dtype_dec_fixed, dtype_dec_fixed, dtype_dec_fixed, + // dtype_int128 + {dtype_unknown, dtype_int128, dtype_int128, dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, - dtype_dec_fixed, dtype_dec_fixed, DTYPE_CANNOT, dtype_double, + dtype_int128, dtype_int128, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, - DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec_fixed, + DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_sql_time_tz {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -909,7 +909,7 @@ const BYTE DSC_multiply_blr4_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double dtype_d_float dtype_sql_date dtype_sql_time dtype_timestamp dtype_blob dtype_array dtype_int64 dtype_dbkey dtype_boolean dtype_dec64 dtype_dec128 - dtype_dec_fixed dtype_sql_time_tz dtype_timestamp_tz + dtype_int128 dtype_sql_time_tz dtype_timestamp_tz */ // dtype_unknown @@ -928,7 +928,7 @@ const BYTE DSC_multiply_blr4_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_cstring {dtype_unknown, dtype_double, dtype_double, dtype_double, @@ -937,7 +937,7 @@ const BYTE DSC_multiply_blr4_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_varying {dtype_unknown, dtype_double, dtype_double, dtype_double, @@ -946,7 +946,7 @@ const BYTE DSC_multiply_blr4_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // 4 (unused) {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -991,7 +991,7 @@ const BYTE DSC_multiply_blr4_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_long {dtype_unknown, dtype_double, dtype_double, dtype_double, @@ -1000,7 +1000,7 @@ const BYTE DSC_multiply_blr4_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_quad {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -1090,7 +1090,7 @@ const BYTE DSC_multiply_blr4_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_dbkey {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -1128,14 +1128,14 @@ const BYTE DSC_multiply_blr4_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT}, - // dtype_dec_fixed - {dtype_unknown, dtype_dec_fixed, dtype_dec_fixed, dtype_dec_fixed, + // dtype_int128 + {dtype_unknown, dtype_int128, dtype_int128, dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, - dtype_dec_fixed, dtype_dec_fixed, DTYPE_CANNOT, dtype_double, + dtype_int128, dtype_int128, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, - DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec_fixed, + DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, - dtype_dec_fixed, DTYPE_CANNOT, DTYPE_CANNOT}, + dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_sql_time_tz {DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, @@ -1268,9 +1268,9 @@ bool DSC_make_descriptor(DSC* desc, desc->dsc_dtype = dtype_dec128; break; - case blr_dec_fixed: - desc->dsc_length = sizeof(DecimalFixed); - desc->dsc_dtype = dtype_dec_fixed; + case blr_int128: + desc->dsc_length = sizeof(Int128); + desc->dsc_dtype = dtype_int128; break; case blr_timestamp: @@ -1448,7 +1448,7 @@ const char* dsc::typeToText() const return "decfloat(16)"; case dtype_dec128: return "decfloat(34)"; - case dtype_dec_fixed: + case dtype_int128: return "decimal"; case dtype_sql_time_tz: return "time with timezone"; @@ -1552,8 +1552,8 @@ void dsc::getSqlInfo(SLONG* sqlLength, SLONG* sqlSubType, SLONG* sqlScale, SLONG *sqlScale = 0; break; - case dtype_dec_fixed: - *sqlType = SQL_DEC_FIXED; + case dtype_int128: + *sqlType = SQL_INT128; *sqlScale = dsc_scale; if (dsc_sub_type) *sqlSubType = dsc_sub_type; diff --git a/src/common/dsc.h b/src/common/dsc.h index b7b78d078b..228ffbda6c 100644 --- a/src/common/dsc.h +++ b/src/common/dsc.h @@ -32,6 +32,7 @@ #include "../jrd/ods.h" #include "../intl/charsets.h" #include "../common/DecFloat.h" +#include "../common/Int128.h" // Data type information @@ -60,7 +61,7 @@ inline bool DTYPE_IS_BLOB_OR_QUAD(UCHAR d) // Exact numeric? inline bool DTYPE_IS_EXACT(UCHAR d) { - return d == dtype_int64 || d == dtype_long || d == dtype_short || d == dtype_dec_fixed; + return d == dtype_int64 || d == dtype_long || d == dtype_short || d == dtype_int128; } inline bool DTYPE_IS_APPROX(UCHAR d) @@ -70,12 +71,13 @@ inline bool DTYPE_IS_APPROX(UCHAR d) inline bool DTYPE_IS_DECFLOAT(UCHAR d) { - return d == dtype_dec128 || d == dtype_dec64 || d == dtype_dec_fixed; + return d == dtype_dec128 || d == dtype_dec64; } inline bool DTYPE_IS_NUMERIC(UCHAR d) { - return (d >= dtype_byte && d <= dtype_d_float) || d == dtype_int64 || DTYPE_IS_DECFLOAT(d); + return (d >= dtype_byte && d <= dtype_d_float) || d == dtype_int64 || + d == dtype_int128 || DTYPE_IS_DECFLOAT(d); } // Descriptor format @@ -138,7 +140,8 @@ typedef struct dsc bool isExact() const { - return dsc_dtype == dtype_int64 || dsc_dtype == dtype_long || dsc_dtype == dtype_short; + return dsc_dtype == dtype_int128 || dsc_dtype == dtype_int64 || + dsc_dtype == dtype_long || dsc_dtype == dtype_short; } bool isNumeric() const @@ -187,14 +190,14 @@ typedef struct dsc return dsc_dtype == dtype_dec128 || dsc_dtype == dtype_dec64; } - bool isDecFixed() const + bool isInt128() const { - return dsc_dtype == dtype_dec_fixed; + return dsc_dtype == dtype_int128; } bool isDecOrInt() const { - return isDecFloat() || isDecFixed() || isExact(); + return isDecFloat() || isExact(); } bool isApprox() const @@ -339,6 +342,15 @@ typedef struct dsc dsc_address = (UCHAR*) address; } + void makeInt128(SCHAR scale, Firebird::Int128* address = NULL) + { + clear(); + dsc_dtype = dtype_int128; + dsc_length = sizeof(Firebird::Int128); + dsc_scale = scale; + dsc_address = (UCHAR*) address; + } + void makeLong(SCHAR scale, SLONG* address = NULL) { clear(); diff --git a/src/common/keywords.cpp b/src/common/keywords.cpp index 6d667d8015..17a4b70718 100644 --- a/src/common/keywords.cpp +++ b/src/common/keywords.cpp @@ -252,6 +252,7 @@ static const TOK tokens[] = {TOK_INSERT, "INSERT", false}, {TOK_INSERTING, "INSERTING", false}, {TOK_INT, "INT", false}, + {TOK_INT128, "INT128", true}, {TOK_INTEGER, "INTEGER", false}, {TOK_INTO, "INTO", false}, {TOK_INVOKER, "INVOKER", true}, diff --git a/src/common/sdl.cpp b/src/common/sdl.cpp index 3080f4dbd7..af220e55b2 100644 --- a/src/common/sdl.cpp +++ b/src/common/sdl.cpp @@ -855,9 +855,9 @@ static const UCHAR* sdl_desc(const UCHAR* ptr, DSC* desc) desc->dsc_length = sizeof(Decimal128); break; - case blr_dec_fixed: - desc->dsc_dtype = dtype_dec_fixed; - desc->dsc_length = sizeof(DecimalFixed); + case blr_int128: + desc->dsc_dtype = dtype_int128; + desc->dsc_length = sizeof(Int128); break; case blr_timestamp: diff --git a/src/common/utils.cpp b/src/common/utils.cpp index f4ecba2260..4a236f6dc1 100644 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -1522,8 +1522,8 @@ UCHAR sqlTypeToDscType(SSHORT sqlType) return dtype_dec64; case SQL_DEC34: return dtype_dec128; - case SQL_DEC_FIXED: - return dtype_dec_fixed; + case SQL_INT128: + return dtype_int128; case SQL_TIME_TZ: return dtype_sql_time_tz; case SQL_TIMESTAMP_TZ: diff --git a/src/common/xdr.cpp b/src/common/xdr.cpp index 4c4c64d8ec..ac53a6e46c 100644 --- a/src/common/xdr.cpp +++ b/src/common/xdr.cpp @@ -31,6 +31,7 @@ #include "../yvalve/gds_proto.h" #include "../common/gdsassert.h" #include "../common/DecFloat.h" +#include "../common/Int128.h" inline UCHAR* XDR_ALLOC(ULONG size) { @@ -268,12 +269,17 @@ bool_t xdr_datum( XDR* xdrs, const dsc* desc, UCHAR* buffer) break; case dtype_dec128: - case dtype_dec_fixed: fb_assert(desc->dsc_length >= sizeof(Firebird::Decimal128)); if (!xdr_dec128(xdrs, reinterpret_cast(p))) return FALSE; break; + case dtype_int128: + fb_assert(desc->dsc_length >= sizeof(Firebird::Int128)); + if (!xdr_int128(xdrs, reinterpret_cast(p))) + return FALSE; + break; + case dtype_timestamp: fb_assert(desc->dsc_length >= 2 * sizeof(SLONG)); if (!xdr_long(xdrs, &((SLONG*) p)[0])) @@ -381,6 +387,19 @@ bool_t xdr_dec128(XDR* xdrs, Firebird::Decimal128* ip) } +bool_t xdr_int128(XDR* xdrs, Firebird::Int128* ip) +{ + UCHAR* bytes = ip->getBytes(); + +#ifndef WORDS_BIGENDIAN + return xdr_hyper(xdrs, &bytes[8]) && xdr_hyper(xdrs, &bytes[0]); +#else + fb_assert(false); // Dec64/128 XDR not tested on bigendians! + return xdr_hyper(xdrs, &bytes[0]) && xdr_hyper(xdrs, &bytes[8]); +#endif +} + + bool_t xdr_enum(XDR* xdrs, xdr_op* ip) { /************************************** diff --git a/src/common/xdr_proto.h b/src/common/xdr_proto.h index b9387e5363..136f81411d 100644 --- a/src/common/xdr_proto.h +++ b/src/common/xdr_proto.h @@ -29,8 +29,9 @@ bool_t xdr_datum(XDR*, const dsc*, UCHAR*); bool_t xdr_double(XDR*, double*); -bool_t xdr_dec64(XDR*, Firebird::Decimal64*); -bool_t xdr_dec128(XDR*, Firebird::Decimal128*); +bool_t xdr_dec64(XDR*, Firebird::Decimal64*); +bool_t xdr_dec128(XDR*, Firebird::Decimal128*); +bool_t xdr_int128(XDR*, Firebird::Int128*); bool_t xdr_enum(XDR*, xdr_op*); bool_t xdr_float(XDR*, float*); bool_t xdr_int(XDR*, int*); diff --git a/src/dsql/AggNodes.cpp b/src/dsql/AggNodes.cpp index 067d31193f..94fec8a07a 100644 --- a/src/dsql/AggNodes.cpp +++ b/src/dsql/AggNodes.cpp @@ -540,6 +540,11 @@ void AvgAggNode::make(DsqlCompilerScratch* dsqlScratch, dsc* desc) ERRD_post(Arg::Gds(isc_expression_eval_err) << Arg::Gds(isc_dsql_agg2_wrongarg) << Arg::Str("AVG")); } + else if (desc->dsc_dtype == dtype_int64 || desc->dsc_dtype == dtype_int128) + { + desc->dsc_dtype = dtype_int128; + desc->dsc_length = sizeof(Int128); + } else if (DTYPE_IS_EXACT(desc->dsc_dtype)) { desc->dsc_dtype = dtype_int64; @@ -584,13 +589,10 @@ void AvgAggNode::getDesc(thread_db* tdbb, CompilerScratch* csb, dsc* desc) return; } - // In V6, the average of an exact type is computed in SINT64, rather than double as in prior - // releases. switch (desc->dsc_dtype) { case dtype_short: case dtype_long: - case dtype_int64: desc->dsc_dtype = dtype_int64; desc->dsc_length = sizeof(SINT64); desc->dsc_sub_type = 0; @@ -598,6 +600,15 @@ void AvgAggNode::getDesc(thread_db* tdbb, CompilerScratch* csb, dsc* desc) nodScale = desc->dsc_scale; break; + case dtype_int64: + case dtype_int128: + desc->dsc_dtype = dtype_int128; + desc->dsc_length = sizeof(Int128); + desc->dsc_sub_type = 0; + desc->dsc_flags = 0; + nodScale = desc->dsc_scale; + break; + case dtype_unknown: desc->dsc_dtype = dtype_unknown; desc->dsc_length = 0; @@ -697,12 +708,19 @@ dsc* AvgAggNode::aggExecute(thread_db* tdbb, jrd_req* request) const SINT64 i; double d; Decimal128 dec; + Int128 i128; if (!dialect1 && impure->vlu_desc.dsc_dtype == dtype_int64) { i = *((SINT64*) impure->vlu_desc.dsc_address) / impure->vlux_count; temp.makeInt64(impure->vlu_desc.dsc_scale, &i); } + else if (!dialect1 && impure->vlu_desc.dsc_dtype == dtype_int128) + { + i128.set(impure->vlux_count, 0); + i128 = ((Int128*) impure->vlu_desc.dsc_address)->div(i128, 0); + temp.makeInt128(impure->vlu_desc.dsc_scale, &i128); + } else if (DTYPE_IS_DECFLOAT(impure->vlu_desc.dsc_dtype)) { DecimalStatus decSt = tdbb->getAttachment()->att_dec_status; @@ -1037,6 +1055,11 @@ void SumAggNode::make(DsqlCompilerScratch* dsqlScratch, dsc* desc) ERRD_post(Arg::Gds(isc_expression_eval_err) << Arg::Gds(isc_dsql_agg2_wrongarg) << Arg::Str("SUM")); } + else if (desc->dsc_dtype == dtype_int64 || desc->dsc_dtype == dtype_int128) + { + desc->dsc_dtype = dtype_int128; + desc->dsc_length = sizeof(Int128); + } else if (DTYPE_IS_EXACT(desc->dsc_dtype)) { desc->dsc_dtype = dtype_int64; @@ -1118,13 +1141,21 @@ void SumAggNode::getDesc(thread_db* tdbb, CompilerScratch* csb, dsc* desc) { case dtype_short: case dtype_long: - case dtype_int64: desc->dsc_dtype = dtype_int64; desc->dsc_length = sizeof(SINT64); nodScale = desc->dsc_scale; desc->dsc_flags = 0; return; + case dtype_int64: + case dtype_int128: + desc->dsc_dtype = dtype_int128; + desc->dsc_length = sizeof(Int128); + desc->dsc_sub_type = 0; + desc->dsc_flags = 0; + nodScale = desc->dsc_scale; + return; + case dtype_unknown: desc->dsc_dtype = dtype_unknown; desc->dsc_length = 0; diff --git a/src/dsql/DdlNodes.epp b/src/dsql/DdlNodes.epp index 59bb0a6c98..31e45ae4a1 100644 --- a/src/dsql/DdlNodes.epp +++ b/src/dsql/DdlNodes.epp @@ -4491,7 +4491,7 @@ void AlterDomainNode::checkUpdate(const dyn_fld& origFld, const dyn_fld& newFld) case blr_float: case blr_dec64: case blr_dec128: - case blr_dec_fixed: + case blr_int128: // Cannot convert column %s from character to non-character data. errorCode = isc_dyn_dtype_conv_invalid; break; @@ -4613,7 +4613,7 @@ void AlterDomainNode::checkUpdate(const dyn_fld& origFld, const dyn_fld& newFld) case blr_float: case blr_dec64: case blr_dec128: - case blr_dec_fixed: + case blr_int128: switch (newFld.dyn_dtype) { case blr_blob: @@ -4741,7 +4741,7 @@ void AlterDomainNode::checkUpdate(const dyn_fld& origFld, const dyn_fld& newFld) case blr_double: case blr_dec64: case blr_dec128: - case blr_dec_fixed: + case blr_int128: break; default: @@ -4751,13 +4751,13 @@ void AlterDomainNode::checkUpdate(const dyn_fld& origFld, const dyn_fld& newFld) } break; - case blr_dec_fixed: + case blr_int128: switch (origFld.dyn_dtype) { case blr_short: case blr_long: case blr_int64: - case blr_dec_fixed: + case blr_int128: break; default: diff --git a/src/dsql/ExprNodes.cpp b/src/dsql/ExprNodes.cpp index dd75a5d6b7..b7d2263625 100644 --- a/src/dsql/ExprNodes.cpp +++ b/src/dsql/ExprNodes.cpp @@ -506,17 +506,19 @@ namespace const UCHAR DSC_ZTYPE_FLT64 = 0; const UCHAR DSC_ZTYPE_FLT128 = 1; const UCHAR DSC_ZTYPE_FIXED = 2; -const UCHAR DSC_ZTYPE_INT = 3; -const UCHAR DSC_ZTYPE_OTHER = 4; -const UCHAR DSC_ZTYPE_BAD = 5; +const UCHAR DSC_ZTYPE_INT64 = 3; +const UCHAR DSC_ZTYPE_INT = 4; +const UCHAR DSC_ZTYPE_OTHER = 5; +const UCHAR DSC_ZTYPE_BAD = 6; -const UCHAR decimalDescTable[5][5] = { -/* DSC_ZTYPE_FLT64 DSC_ZTYPE_FLT128 DSC_ZTYPE_FIXED DSC_ZTYPE_INT DSC_ZTYPE_OTHER */ -/* DSC_ZTYPE_FLT64 */ {DSC_ZTYPE_FLT64, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128}, -/* DSC_ZTYPE_FLT128 */ {DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128}, -/* DSC_ZTYPE_FIXED */ {DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FIXED, DSC_ZTYPE_FIXED, DSC_ZTYPE_FLT128}, -/* DSC_ZTYPE_INT */ {DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FIXED, DSC_ZTYPE_BAD, DSC_ZTYPE_BAD}, -/* DSC_ZTYPE_OTHER */ {DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_BAD, DSC_ZTYPE_BAD} +const UCHAR decimalDescTable[6][6] = { +/* DSC_ZTYPE_FLT64 DSC_ZTYPE_FLT128 DSC_ZTYPE_FIXED DSC_ZTYPE_INT64 DSC_ZTYPE_INT DSC_ZTYPE_OTHER */ +/* DSC_ZTYPE_FLT64 */ {DSC_ZTYPE_FLT64, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128}, +/* DSC_ZTYPE_FLT128 */ {DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128}, +/* DSC_ZTYPE_FIXED */ {DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FIXED, DSC_ZTYPE_FIXED, DSC_ZTYPE_FIXED, DSC_ZTYPE_FLT128}, +/* DSC_ZTYPE_INT64 */ {DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FIXED, DSC_ZTYPE_FIXED, DSC_ZTYPE_BAD, DSC_ZTYPE_BAD}, +/* DSC_ZTYPE_INT */ {DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FIXED, DSC_ZTYPE_BAD, DSC_ZTYPE_BAD, DSC_ZTYPE_BAD}, +/* DSC_ZTYPE_OTHER */ {DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_FLT128, DSC_ZTYPE_BAD, DSC_ZTYPE_BAD, DSC_ZTYPE_BAD} }; UCHAR getFType(const dsc& desc) @@ -527,8 +529,10 @@ UCHAR getFType(const dsc& desc) return DSC_ZTYPE_FLT64; case dtype_dec128: return DSC_ZTYPE_FLT128; - case dtype_dec_fixed: + case dtype_int128: return DSC_ZTYPE_FIXED; + case dtype_int64: + return DSC_ZTYPE_INT64; } if (DTYPE_IS_EXACT(desc.dsc_dtype)) @@ -547,7 +551,7 @@ unsigned setDecDesc(dsc* desc, const dsc& desc1, const dsc& desc2, Scaling sc, S zipType = DSC_ZTYPE_FLT128; // In production case fallback to Decimal128 desc->dsc_dtype = zipType == DSC_ZTYPE_FLT64 ? dtype_dec64 : - zipType == DSC_ZTYPE_FLT128 ? dtype_dec128 : dtype_dec_fixed; + zipType == DSC_ZTYPE_FLT128 ? dtype_dec128 : dtype_int128; desc->dsc_sub_type = 0; desc->dsc_flags = (desc1.dsc_flags | desc2.dsc_flags) & DSC_nullable; desc->dsc_scale = 0; @@ -569,9 +573,9 @@ unsigned setDecDesc(dsc* desc, const dsc& desc1, const dsc& desc2, Scaling sc, S *nodScale = desc->dsc_scale; desc->dsc_length = zipType == DSC_ZTYPE_FLT64 ? sizeof(Decimal64) : - zipType == DSC_ZTYPE_FLT128 ? sizeof(Decimal128) : sizeof(DecimalFixed); + zipType == DSC_ZTYPE_FLT128 ? sizeof(Decimal128) : sizeof(Int128); - return zipType == DSC_ZTYPE_FIXED ? ExprNode::FLAG_DECFIXED : ExprNode::FLAG_DECFLOAT; + return zipType == DSC_ZTYPE_FIXED ? ExprNode::FLAG_INT128 : ExprNode::FLAG_DECFLOAT; } } // anon namespace @@ -764,7 +768,7 @@ void ArithmeticNode::makeDialect1(dsc* desc, dsc& desc1, dsc& desc2) case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: setDecDesc(desc, desc1, desc2, SCALE_MIN); break; @@ -792,7 +796,7 @@ void ArithmeticNode::makeDialect1(dsc* desc, dsc& desc1, dsc& desc2) switch (dtype) { case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: setDecDesc(desc, desc1, desc2, SCALE_SUM); break; @@ -887,8 +891,8 @@ void ArithmeticNode::makeDialect3(dsc* desc, dsc& desc1, dsc& desc2) // arithmetic, but returns a if (DTYPE_IS_EXACT(dtype1) && DTYPE_IS_EXACT(dtype2)) { - if (desc1.isDecFixed() || desc2.isDecFixed()) - dtype = dtype_dec_fixed; + if (desc1.isInt128() || desc2.isInt128()) + dtype = dtype_int128; else dtype = dtype_int64; } @@ -1027,7 +1031,7 @@ void ArithmeticNode::makeDialect3(dsc* desc, dsc& desc1, dsc& desc2) case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: setDecDesc(desc, desc1, desc2, SCALE_MIN); break; @@ -1076,7 +1080,7 @@ void ArithmeticNode::makeDialect3(dsc* desc, dsc& desc1, dsc& desc2) switch (dtype) { - case dtype_dec_fixed: + case dtype_int128: case dtype_dec128: setDecDesc(desc, desc1, desc2, SCALE_SUM); break; @@ -1135,7 +1139,7 @@ void ArithmeticNode::makeDialect3(dsc* desc, dsc& desc1, dsc& desc2) break; case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: setDecDesc(desc, desc1, desc2, SCALE_SUM); break; @@ -1347,7 +1351,7 @@ void ArithmeticNode::getDescDialect1(thread_db* /*tdbb*/, dsc* desc, dsc& desc1, case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: nodFlags |= setDecDesc(desc, desc1, desc2, SCALE_MIN, &nodScale); break; @@ -1394,7 +1398,7 @@ void ArithmeticNode::getDescDialect1(thread_db* /*tdbb*/, dsc* desc, dsc& desc1, return; case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: nodFlags |= setDecDesc(desc, desc1, desc2, SCALE_SUM, &nodScale); break; @@ -1469,8 +1473,8 @@ void ArithmeticNode::getDescDialect3(thread_db* /*tdbb*/, dsc* desc, dsc& desc1, if (DTYPE_IS_EXACT(dtype1) && DTYPE_IS_EXACT(dtype2)) { - if (desc1.isDecFixed() || desc2.isDecFixed()) - dtype = dtype_dec_fixed; + if (desc1.isInt128() || desc2.isInt128()) + dtype = dtype_int128; else dtype = dtype_int64; } @@ -1620,7 +1624,7 @@ void ArithmeticNode::getDescDialect3(thread_db* /*tdbb*/, dsc* desc, dsc& desc1, case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: nodFlags |= setDecDesc(desc, desc1, desc2, SCALE_MIN, &nodScale); return; @@ -1674,7 +1678,7 @@ void ArithmeticNode::getDescDialect3(thread_db* /*tdbb*/, dsc* desc, dsc& desc1, return; case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: nodFlags |= setDecDesc(desc, desc1, desc2, SCALE_SUM, &nodScale); return; @@ -1900,23 +1904,6 @@ dsc* ArithmeticNode::add(thread_db* tdbb, const dsc* desc, impure_value* value, return result; } - if (node->nodFlags & FLAG_DECFIXED) - { - const DecimalFixed d1 = MOV_get_dec_fixed(tdbb, desc, node->nodScale); - const DecimalFixed d2 = MOV_get_dec_fixed(tdbb, &value->vlu_desc, node->nodScale); - - DecimalStatus decSt = tdbb->getAttachment()->att_dec_status; - value->vlu_misc.vlu_dec_fixed = (blrOp == blr_subtract) ? d2.sub(decSt, d1) : d1.add(decSt, d2); - - result->dsc_dtype = dtype_dec_fixed; - result->dsc_length = sizeof(DecimalFixed); - result->dsc_scale = node->nodScale; - result->dsc_sub_type = 0; - result->dsc_address = (UCHAR*) &value->vlu_misc.vlu_dec_fixed; - - return result; - } - // Handle floating arithmetic if (node->nodFlags & FLAG_DOUBLE) @@ -1996,19 +1983,20 @@ dsc* ArithmeticNode::add2(thread_db* tdbb, const dsc* desc, impure_value* value, return result; } - if (node->nodFlags & FLAG_DECFIXED) + // 128-bit arithmetic + + if (node->nodFlags & FLAG_INT128) { - const DecimalFixed d1 = MOV_get_dec_fixed(tdbb, desc, node->nodScale); - const DecimalFixed d2 = MOV_get_dec_fixed(tdbb, &value->vlu_desc, node->nodScale); + const Int128 d1 = MOV_get_int128(tdbb, desc, node->nodScale); + const Int128 d2 = MOV_get_int128(tdbb, &value->vlu_desc, node->nodScale); - DecimalStatus decSt = tdbb->getAttachment()->att_dec_status; - value->vlu_misc.vlu_dec_fixed = (blrOp == blr_subtract) ? d2.sub(decSt, d1) : d1.add(decSt, d2); + value->vlu_misc.vlu_int128 = (blrOp == blr_subtract) ? d2.sub(d1) : d1.add(d2); - result->dsc_dtype = dtype_dec_fixed; - result->dsc_length = sizeof(DecimalFixed); + result->dsc_dtype = dtype_int128; + result->dsc_length = sizeof(Int128); result->dsc_scale = node->nodScale; result->dsc_sub_type = 0; - result->dsc_address = (UCHAR*) &value->vlu_misc.vlu_dec_fixed; + result->dsc_address = (UCHAR*) &value->vlu_misc.vlu_int128; return result; } @@ -2099,19 +2087,20 @@ dsc* ArithmeticNode::multiply(const dsc* desc, impure_value* value) const return &value->vlu_desc; } - if (nodFlags & FLAG_DECFIXED) + // 128-bit arithmetic + + if (nodFlags & FLAG_INT128) { - const DecimalFixed d1 = MOV_get_dec_fixed(tdbb, desc, nodScale); - const DecimalFixed d2 = MOV_get_dec_fixed(tdbb, &value->vlu_desc, nodScale); + const Int128 d1 = MOV_get_int128(tdbb, desc, nodScale); + const Int128 d2 = MOV_get_int128(tdbb, &value->vlu_desc, nodScale); - DecimalStatus decSt = tdbb->getAttachment()->att_dec_status; - value->vlu_misc.vlu_dec_fixed = d1.mul(decSt, d2); + value->vlu_misc.vlu_int128 = d1.mul(d2); - value->vlu_desc.dsc_dtype = dtype_dec_fixed; - value->vlu_desc.dsc_length = sizeof(DecimalFixed); + value->vlu_desc.dsc_dtype = dtype_int128; + value->vlu_desc.dsc_length = sizeof(Int128); value->vlu_desc.dsc_scale = nodScale; value->vlu_desc.dsc_sub_type = 0; - value->vlu_desc.dsc_address = (UCHAR*) &value->vlu_misc.vlu_dec_fixed; + value->vlu_desc.dsc_address = (UCHAR*) &value->vlu_misc.vlu_int128; return &value->vlu_desc; } @@ -2210,20 +2199,21 @@ dsc* ArithmeticNode::multiply2(const dsc* desc, impure_value* value) const return &value->vlu_desc; } - if (nodFlags & FLAG_DECFIXED) + // 128-bit arithmetic + + if (nodFlags & FLAG_INT128) { const SSHORT scale = NUMERIC_SCALE(*desc); - const DecimalFixed d1 = MOV_get_dec_fixed(tdbb, desc, scale); - const DecimalFixed d2 = MOV_get_dec_fixed(tdbb, &value->vlu_desc, nodScale - scale); + const Int128 d1 = MOV_get_int128(tdbb, desc, scale); + const Int128 d2 = MOV_get_int128(tdbb, &value->vlu_desc, nodScale - scale); - DecimalStatus decSt = tdbb->getAttachment()->att_dec_status; - value->vlu_misc.vlu_dec_fixed = d1.mul(decSt, d2); + value->vlu_misc.vlu_int128 = d1.mul(d2); - value->vlu_desc.dsc_dtype = dtype_dec_fixed; - value->vlu_desc.dsc_length = sizeof(DecimalFixed); + value->vlu_desc.dsc_dtype = dtype_int128; + value->vlu_desc.dsc_length = sizeof(Int128); value->vlu_desc.dsc_scale = nodScale; value->vlu_desc.dsc_sub_type = 0; - value->vlu_desc.dsc_address = (UCHAR*) &value->vlu_misc.vlu_dec_fixed; + value->vlu_desc.dsc_address = (UCHAR*) &value->vlu_misc.vlu_int128; return &value->vlu_desc; } @@ -2324,20 +2314,21 @@ dsc* ArithmeticNode::divide2(const dsc* desc, impure_value* value) const return &value->vlu_desc; } - if (nodFlags & FLAG_DECFIXED) + // 128-bit arithmetic + + if (nodFlags & FLAG_INT128) { const SSHORT scale = NUMERIC_SCALE(*desc); - const DecimalFixed d2 = MOV_get_dec_fixed(tdbb, desc, scale); - const DecimalFixed d1 = MOV_get_dec_fixed(tdbb, &value->vlu_desc, nodScale - scale); + const Int128 d2 = MOV_get_int128(tdbb, desc, scale); + const Int128 d1 = MOV_get_int128(tdbb, &value->vlu_desc, nodScale - scale); - DecimalStatus decSt = tdbb->getAttachment()->att_dec_status; - value->vlu_misc.vlu_dec_fixed = d1.div(decSt, d2, scale * 2); + value->vlu_misc.vlu_int128 = d1.div(d2, scale * 2); - value->vlu_desc.dsc_dtype = dtype_dec_fixed; - value->vlu_desc.dsc_length = sizeof(DecimalFixed); + value->vlu_desc.dsc_dtype = dtype_int128; + value->vlu_desc.dsc_length = sizeof(Int128); value->vlu_desc.dsc_scale = nodScale; value->vlu_desc.dsc_sub_type = 0; - value->vlu_desc.dsc_address = (UCHAR*) &value->vlu_misc.vlu_dec_fixed; + value->vlu_desc.dsc_address = (UCHAR*) &value->vlu_misc.vlu_int128; return &value->vlu_desc; } @@ -7383,18 +7374,17 @@ DmlNode* LiteralNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* case dtype_double: case dtype_dec128: { - SSHORT scale; - UCHAR dtype; - // The double literal could potentially be used for any numeric literal - the value is // passed as if it were a text string. Convert the numeric string to its binary value // (int64, long or double as appropriate). l = csb->csb_blr_reader.getWord(); q = csb->csb_blr_reader.getPos(); - dtype = CVT_get_numeric(q, l, &scale, p); + SSHORT scale = 0; + UCHAR dtype = CVT_get_numeric(q, l, &scale, p); node->litDesc.dsc_dtype = dtype; node->dsqlStr = FB_NEW_POOL(pool) IntlString(pool, string(q, l)); + node->litDesc.dsc_scale = (SCHAR) scale; switch (dtype) { @@ -7406,11 +7396,9 @@ DmlNode* LiteralNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* break; case dtype_long: node->litDesc.dsc_length = sizeof(SLONG); - node->litDesc.dsc_scale = (SCHAR) scale; break; default: node->litDesc.dsc_length = sizeof(SINT64); - node->litDesc.dsc_scale = (SCHAR) scale; } break; } @@ -7846,25 +7834,34 @@ bool LiteralNode::sameAs(CompilerScratch* csb, const ExprNode* other, bool ignor ValueExprNode* LiteralNode::pass2(thread_db* tdbb, CompilerScratch* csb) { - if (DTYPE_IS_DECFLOAT(csb->csb_preferredDataType) && dsqlStr) + if (csb->csb_preferredDesc && ((DTYPE_IS_DECFLOAT(csb->csb_preferredDesc->dsc_dtype) || + csb->csb_preferredDesc->dsc_dtype == dtype_int128) && dsqlStr)) { const string& s(dsqlStr->getString()); dsc desc; desc.makeText(s.length(), CS_ASCII, (UCHAR*) s.c_str()); - switch (csb->csb_preferredDataType) + switch (csb->csb_preferredDesc->dsc_dtype) { case dtype_dec64: *((Decimal64*) litDesc.dsc_address) = CVT_get_dec64(&desc, tdbb->getAttachment()->att_dec_status, ERR_post); litDesc.dsc_dtype = dtype_dec64; + litDesc.dsc_scale = 0; break; case dtype_dec128: - case dtype_dec_fixed: *((Decimal128*) litDesc.dsc_address) = CVT_get_dec128(&desc, tdbb->getAttachment()->att_dec_status, ERR_post); litDesc.dsc_dtype = dtype_dec128; + litDesc.dsc_scale = 0; + break; + + case dtype_int128: + *((Int128*) litDesc.dsc_address) = CVT_get_int128(&desc, + csb->csb_preferredDesc->dsc_scale, tdbb->getAttachment()->att_dec_status, ERR_post); + litDesc.dsc_dtype = dtype_int128; + litDesc.dsc_scale = csb->csb_preferredDesc->dsc_scale; break; } } @@ -8744,8 +8741,8 @@ dsc* NegateNode::execute(thread_db* tdbb, jrd_req* request) const impure->vlu_misc.vlu_dec128 = impure->vlu_misc.vlu_dec128.neg(); break; - case dtype_dec_fixed: - impure->vlu_misc.vlu_dec_fixed = impure->vlu_misc.vlu_dec_fixed.neg(); + case dtype_int128: + impure->vlu_misc.vlu_int128 = impure->vlu_misc.vlu_int128.neg(); break; case dtype_int64: @@ -11091,7 +11088,7 @@ void SubQueryNode::getDesc(thread_db* tdbb, CompilerScratch* csb, dsc* desc) case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: desc->dsc_dtype = dtype_dec128; desc->dsc_length = sizeof(Decimal128); desc->dsc_scale = 0; diff --git a/src/dsql/Nodes.h b/src/dsql/Nodes.h index 255953724b..aa6a4a3993 100644 --- a/src/dsql/Nodes.h +++ b/src/dsql/Nodes.h @@ -532,7 +532,7 @@ public: static const unsigned FLAG_DATE = 0x20; static const unsigned FLAG_DECFLOAT = 0x40; static const unsigned FLAG_VALUE = 0x80; // Full value area required in impure space. - static const unsigned FLAG_DECFIXED = 0x100; + static const unsigned FLAG_INT128 = 0x100; explicit ExprNode(Type aType, MemoryPool& pool) : DmlNode(pool), diff --git a/src/dsql/StmtNodes.cpp b/src/dsql/StmtNodes.cpp index 198d7ed649..0fefb75f62 100644 --- a/src/dsql/StmtNodes.cpp +++ b/src/dsql/StmtNodes.cpp @@ -501,7 +501,7 @@ AssignmentNode* AssignmentNode::pass2(thread_db* tdbb, CompilerScratch* csb) { // scope dsc desc; asgnTo->getDesc(tdbb, csb, &desc); - AutoSetRestore dataType(&csb->csb_preferredDataType, desc.dsc_dtype); + AutoSetRestore dataType(&csb->csb_preferredDesc, &desc); ExprNode::doPass2(tdbb, csb, asgnFrom.getAddress()); } ExprNode::doPass2(tdbb, csb, asgnTo.getAddress()); @@ -8360,11 +8360,14 @@ void SetDecFloatTrapsNode::execute(thread_db* tdbb, dsql_req* /*request*/, jrd_t //-------------------- -void SetDecFloatBindNode::execute(thread_db* tdbb, dsql_req* /*request*/, jrd_tra** /*traHandle*/) const +void SetHighPrecBindNode::execute(thread_db* tdbb, dsql_req* /*request*/, jrd_tra** /*traHandle*/) const { SET_TDBB(tdbb); Attachment* const attachment = tdbb->getAttachment(); - attachment->att_dec_binding = bind; + if (bindInt128) + attachment->att_i128_binding = bind; + else + attachment->att_dec_binding = bind; } diff --git a/src/dsql/StmtNodes.h b/src/dsql/StmtNodes.h index cdf6ad91f4..7a9854124f 100644 --- a/src/dsql/StmtNodes.h +++ b/src/dsql/StmtNodes.h @@ -30,6 +30,7 @@ #include "../dsql/Nodes.h" #include "../dsql/DdlNodes.h" #include "../dsql/NodePrinter.h" +#include "../common/DecFloat.h" namespace Jrd { @@ -1748,11 +1749,13 @@ public: }; -class SetDecFloatBindNode : public SessionManagementNode +class SetHighPrecBindNode : public SessionManagementNode { public: - SetDecFloatBindNode(MemoryPool& pool) - : SessionManagementNode(pool) + SetHighPrecBindNode(MemoryPool& pool, bool isInt128) + : SessionManagementNode(pool), + bind(Firebird::NumericBinding::NUM_NATIVE), + bindInt128(isInt128) { } @@ -1764,13 +1767,14 @@ public: NODE_PRINT(printer, bind.bind); NODE_PRINT(printer, bind.numScale); - return "SetDecFloatBindNode"; + return "SetHighPrecBindNode"; } virtual void execute(thread_db* tdbb, dsql_req* request, jrd_tra** traHandle) const; public: - Firebird::DecimalBinding bind; + Firebird::NumericBinding bind; + bool bindInt128; }; diff --git a/src/dsql/ddl_proto.h b/src/dsql/ddl_proto.h index 0743dc947d..56b1963b3d 100644 --- a/src/dsql/ddl_proto.h +++ b/src/dsql/ddl_proto.h @@ -60,7 +60,7 @@ const USHORT blr_dtypes[] = { blr_bool, // dtype_boolean blr_dec64, // dtype_dec64 blr_dec128, // dtype_dec128 - blr_dec_fixed, // dtype_dec_fixed + blr_int128, // dtype_int128 blr_sql_time_tz, // dtype_sql_time_tz blr_timestamp_tz // dtype_timestamp_tz }; diff --git a/src/dsql/dsql.cpp b/src/dsql/dsql.cpp index dcee054350..9bc9ddf9e2 100644 --- a/src/dsql/dsql.cpp +++ b/src/dsql/dsql.cpp @@ -1148,7 +1148,7 @@ void dsql_req::mapInOut(thread_db* tdbb, bool toExternal, const dsql_msg* messag desc.dsc_address = dsql_msg_buf + (IPTR) desc.dsc_address; if (notNull) - MOVD_move(tdbb, &parDesc, &desc, toExternal); + MOVD_move(tdbb, &parDesc, &desc); else memset(desc.dsc_address, 0, desc.dsc_length); } @@ -1156,7 +1156,7 @@ void dsql_req::mapInOut(thread_db* tdbb, bool toExternal, const dsql_msg* messag { // Safe cast because desc is used as source only. desc.dsc_address = const_cast(in_dsql_msg_buf) + (IPTR) desc.dsc_address; - MOVD_move(tdbb, &desc, &parDesc, toExternal); + MOVD_move(tdbb, &desc, &parDesc); } else memset(parDesc.dsc_address, 0, parDesc.dsc_length); @@ -1191,7 +1191,7 @@ void dsql_req::mapInOut(thread_db* tdbb, bool toExternal, const dsql_msg* messag dsc desc = parameter->par_desc; desc.dsc_address = msgBuffer + (IPTR) desc.dsc_address; - MOVD_move(tdbb, &parentDesc, &desc, false); + MOVD_move(tdbb, &parentDesc, &desc); dsql_par* null_ind = parameter->par_null; if (null_ind != NULL) @@ -1221,7 +1221,7 @@ void dsql_req::mapInOut(thread_db* tdbb, bool toExternal, const dsql_msg* messag dsc desc = parameter->par_desc; desc.dsc_address = msgBuffer + (IPTR) desc.dsc_address; - MOVD_move(tdbb, &parentDesc, &desc, false); + MOVD_move(tdbb, &parentDesc, &desc); dsql_par* null_ind = parameter->par_null; if (null_ind != NULL) diff --git a/src/dsql/dsql.h b/src/dsql/dsql.h index e1fcccc593..4b2f1b2c27 100644 --- a/src/dsql/dsql.h +++ b/src/dsql/dsql.h @@ -256,8 +256,8 @@ public: precision = 18; break; - case dtype_dec_fixed: - precision = 34; + case dtype_int128: + precision = 38; break; default: diff --git a/src/dsql/gen.cpp b/src/dsql/gen.cpp index 2a0bd96952..39b631cf97 100644 --- a/src/dsql/gen.cpp +++ b/src/dsql/gen.cpp @@ -203,21 +203,24 @@ void GEN_port(DsqlCompilerScratch* dsqlScratch, dsql_msg* message) if (fromCharSet != toCharSet) parameter->par_desc.setTextType(toCharSet); } - else if (parameter->par_desc.isDecFloat()) + else if (parameter->par_desc.isDecFloat() || + parameter->par_desc.dsc_dtype == dtype_int128) { - const DecimalBinding& b = tdbb->getAttachment()->att_dec_binding; + const NumericBinding& b(parameter->par_desc.dsc_dtype == dtype_int128 ? + tdbb->getAttachment()->att_i128_binding : tdbb->getAttachment()->att_dec_binding); switch (b.bind) { - case DecimalBinding::DEC_NATIVE: + case NumericBinding::NUM_NATIVE: break; - case DecimalBinding::DEC_TEXT: + case NumericBinding::NUM_TEXT: parameter->par_desc.makeText((parameter->par_desc.dsc_dtype == dtype_dec64 ? - IDecFloat16::STRING_SIZE : IDecFloat34::STRING_SIZE) - 1, ttype_ascii); + IDecFloat16::STRING_SIZE : parameter->par_desc.dsc_dtype == dtype_dec128 ? + IDecFloat34::STRING_SIZE : IInt128::STRING_SIZE) - 1, ttype_ascii); break; - case DecimalBinding::DEC_DOUBLE: + case NumericBinding::NUM_DOUBLE: parameter->par_desc.makeDouble(); break; - case DecimalBinding::DEC_NUMERIC: + case NumericBinding::NUM_INT64: parameter->par_desc.makeInt64(b.numScale); break; } @@ -415,8 +418,8 @@ void GEN_descriptor( DsqlCompilerScratch* dsqlScratch, const dsc* desc, bool tex dsqlScratch->appendUChar(blr_dec128); break; - case dtype_dec_fixed: - dsqlScratch->appendUChar(blr_dec_fixed); + case dtype_int128: + dsqlScratch->appendUChar(blr_int128); dsqlScratch->appendUChar(desc->dsc_scale); break; diff --git a/src/dsql/movd.cpp b/src/dsql/movd.cpp index d9fbe42e60..080c1ea744 100644 --- a/src/dsql/movd.cpp +++ b/src/dsql/movd.cpp @@ -33,11 +33,11 @@ using namespace Firebird; // Move (and possible convert) something to something else. -void MOVD_move(thread_db* tdbb, dsc* from, dsc* to, bool toExternal) +void MOVD_move(thread_db* tdbb, dsc* from, dsc* to) { try { - MOV_move_ext(tdbb, from, to, toExternal); + MOV_move(tdbb, from, to); } catch (const status_exception& ex) { diff --git a/src/dsql/movd_proto.h b/src/dsql/movd_proto.h index 5a230b42c7..60f1c18288 100644 --- a/src/dsql/movd_proto.h +++ b/src/dsql/movd_proto.h @@ -24,6 +24,6 @@ #ifndef DSQL_MOVD_PROTO_H #define DSQL_MOVD_PROTO_H -void MOVD_move(Jrd::thread_db* tdbb, dsc*, dsc*, bool toExternal); +void MOVD_move(Jrd::thread_db* tdbb, dsc*, dsc*); #endif // DSQL_MOVD_PROTO_H diff --git a/src/dsql/parse.y b/src/dsql/parse.y index 7362a43b53..6a297f6606 100644 --- a/src/dsql/parse.y +++ b/src/dsql/parse.y @@ -613,6 +613,7 @@ using namespace Firebird; %token HEX_ENCODE %token IDLE %token INVOKER +%token INT128 %token IV %token LAST_DAY %token LEGACY @@ -816,7 +817,7 @@ using namespace Firebird; Jrd::CreateAlterRoleNode* createAlterRoleNode; Jrd::SetDecFloatRoundNode* setDecFloatRoundNode; Jrd::SetDecFloatTrapsNode* setDecFloatTrapsNode; - Jrd::SetDecFloatBindNode* setDecFloatBindNode; + Jrd::SetHighPrecBindNode* setHighPrecBindNode; Jrd::SessionResetNode* sessionResetNode; } @@ -4224,6 +4225,7 @@ keyword_or_column | VAR_SAMP | VAR_POP | DECFLOAT // added in FB 4.0 + | INT128 | LOCAL | LOCALTIME | LOCALTIMESTAMP @@ -4971,14 +4973,14 @@ prec_scale { $$ = newNode(); - if ($2 < 1 || $2 > 34) - yyabandon(YYPOSNARG(2), -842, Arg::Gds(isc_precision_err2) << Arg::Num(1) << Arg::Num(34)); - // Precision must be between 1 and 34 + if ($2 < 1 || $2 > 38) + yyabandon(YYPOSNARG(2), -842, Arg::Gds(isc_precision_err2) << Arg::Num(1) << Arg::Num(38)); + // Precision must be between 1 and 38 if ($2 > 18) { - $$->dtype = dtype_dec_fixed; - $$->length = sizeof(DecimalFixed); + $$->dtype = dtype_int128; + $$->length = sizeof(Int128); } else if ($2 > 9) { @@ -5027,17 +5029,17 @@ prec_scale { $$ = newNode(); - if ($2 < 1 || $2 > 34) - yyabandon(YYPOSNARG(2), -842, Arg::Gds(isc_precision_err2) << Arg::Num(1) << Arg::Num(34)); - // Precision must be between 1 and 34 + if ($2 < 1 || $2 > 38) + yyabandon(YYPOSNARG(2), -842, Arg::Gds(isc_precision_err2) << Arg::Num(1) << Arg::Num(38)); + // Precision must be between 1 and 38 if ($4 > $2 || $4 < 0) yyabandon(YYPOSNARG(4), -842, isc_scale_nogt); // Scale must be between 0 and precision if ($2 > 18) { - $$->dtype = dtype_dec_fixed; - $$->length = sizeof(DecimalFixed); + $$->dtype = dtype_int128; + $$->length = sizeof(Int128); } else if ($2 > 9) { @@ -5270,14 +5272,20 @@ set_decfloat_traps { $$ = $5; } ; -%type set_decfloat_bind +%type set_decfloat_bind set_decfloat_bind - : SET DECFLOAT BIND - { $$ = newNode(); } + : SET bind_to_type BIND + { $$ = newNode($2); } decfloat_bind_clause($4) { $$ = $4; } ; +%type bind_to_type +bind_to_type + : DECFLOAT { $$ = false; } + | INT128 { $$ = true; } + ; + %type decfloat_traps_list_opt() decfloat_traps_list_opt($setDecFloatTrapsNode) : // nothing @@ -5296,26 +5304,26 @@ decfloat_trap($setDecFloatTrapsNode) { $setDecFloatTrapsNode->trap($1); } ; -%type decfloat_bind_clause() -decfloat_bind_clause($setDecFloatBindNode) +%type decfloat_bind_clause() +decfloat_bind_clause($setHighPrecBindNode) : NATIVE // do nothing | character_keyword - { $setDecFloatBindNode->bind.bind = DecimalBinding::DEC_TEXT; } + { $setHighPrecBindNode->bind.bind = NumericBinding::NUM_TEXT; } | DOUBLE PRECISION - { $setDecFloatBindNode->bind.bind = DecimalBinding::DEC_DOUBLE; } - | BIGINT decfloat_scale_clause($setDecFloatBindNode) - { $setDecFloatBindNode->bind.bind = DecimalBinding::DEC_NUMERIC; } + { $setHighPrecBindNode->bind.bind = NumericBinding::NUM_DOUBLE; } + | BIGINT decfloat_scale_clause($setHighPrecBindNode) + { $setHighPrecBindNode->bind.bind = NumericBinding::NUM_INT64; } ; -%type decfloat_scale_clause() -decfloat_scale_clause($setDecFloatBindNode) +%type decfloat_scale_clause() +decfloat_scale_clause($setHighPrecBindNode) : // nothing | ',' signed_long_integer { - if ($2 > DecimalBinding::MAX_SCALE || $2 < 0) + if ($2 > NumericBinding::MAX_SCALE || $2 < 0) yyabandon(YYPOSNARG(2), -842, isc_scale_nogt); // Scale must be between 0 and precision - $setDecFloatBindNode->bind.numScale = -$2; + $setHighPrecBindNode->bind.numScale = -$2; } %type session_statement diff --git a/src/include/firebird/FirebirdInterface.idl b/src/include/firebird/FirebirdInterface.idl index 12952bc182..2bba9ee61d 100644 --- a/src/include/firebird/FirebirdInterface.idl +++ b/src/include/firebird/FirebirdInterface.idl @@ -34,6 +34,7 @@ typedef ISC_TIME_TZ; typedef ISC_TIMESTAMP_TZ; typedef FB_DEC16; typedef FB_DEC34; +typedef FB_I128; typedef isc_tr_handle; typedef isc_stmt_handle; @@ -1093,7 +1094,7 @@ interface Util : Versioned XpbBuilder getXpbBuilder(Status status, uint kind, const uchar* buf, uint len); uint setOffsets(Status status, MessageMetadata metadata, OffsetsCallback callback); -version: // 3.0 => 4.0 +version: // 3.0 => 4.0 Alpha1 DecFloat16 getDecFloat16(Status status); DecFloat34 getDecFloat34(Status status); Transaction getTransactionByHandle(Status status, isc_tr_handle* hndlPtr); @@ -1106,6 +1107,9 @@ version: // 3.0 => 4.0 uint fractions, const string timeZone); void encodeTimeStampTz(Status status, ISC_TIMESTAMP_TZ* timeStampTz, uint year, uint month, uint day, uint hours, uint minutes, uint seconds, uint fractions, const string timeZone); + +version: // 4.0 Beta1 => 4.0 Beta2 + Int128 getInt128(Status status); } interface OffsetsCallback : Versioned @@ -1515,6 +1519,15 @@ interface DecFloat34 : Versioned void fromString(Status status, const string from, FB_DEC34* to); } +interface Int128 : Versioned +{ + // - 170141183460469231731687303715884105728 e - 128 \0 + // 1 + 39 + 1 + 1 + 3 + 1 = 46 + const uint STRING_SIZE = 46; // includes terminating \0 + void toString(Status status, const FB_I128* from, int scale, uint bufferLength, string buffer); + void fromString(Status status, int scale, const string from, FB_I128* to); +} + // Replication interfaces interface ReplicatedRecord : Versioned diff --git a/src/include/firebird/IdlFbInterfaces.h b/src/include/firebird/IdlFbInterfaces.h index d936ede4c9..ae228568d7 100644 --- a/src/include/firebird/IdlFbInterfaces.h +++ b/src/include/firebird/IdlFbInterfaces.h @@ -116,6 +116,7 @@ namespace Firebird class IUdrPlugin; class IDecFloat16; class IDecFloat34; + class IInt128; class IReplicatedRecord; class IReplicatedBlob; class IReplicatedTransaction; @@ -4051,6 +4052,7 @@ namespace Firebird void (CLOOP_CARG *decodeTimeStampTz)(IUtil* self, IStatus* status, const ISC_TIMESTAMP_TZ* timeStampTz, unsigned* year, unsigned* month, unsigned* day, unsigned* hours, unsigned* minutes, unsigned* seconds, unsigned* fractions, unsigned timeZoneBufferLength, char* timeZoneBuffer) throw(); void (CLOOP_CARG *encodeTimeTz)(IUtil* self, IStatus* status, ISC_TIME_TZ* timeTz, unsigned hours, unsigned minutes, unsigned seconds, unsigned fractions, const char* timeZone) throw(); void (CLOOP_CARG *encodeTimeStampTz)(IUtil* self, IStatus* status, ISC_TIMESTAMP_TZ* timeStampTz, unsigned year, unsigned month, unsigned day, unsigned hours, unsigned minutes, unsigned seconds, unsigned fractions, const char* timeZone) throw(); + IInt128* (CLOOP_CARG *getInt128)(IUtil* self, IStatus* status) throw(); }; protected: @@ -4064,7 +4066,7 @@ namespace Firebird } public: - static const unsigned VERSION = 3; + static const unsigned VERSION = 4; template void getFbVersion(StatusType* status, IAttachment* att, IVersionCallback* callback) { @@ -4259,6 +4261,20 @@ namespace Firebird static_cast(this->cloopVTable)->encodeTimeStampTz(this, status, timeStampTz, year, month, day, hours, minutes, seconds, fractions, timeZone); StatusType::checkException(status); } + + template IInt128* getInt128(StatusType* status) + { + if (cloopVTable->version < 4) + { + StatusType::setVersionError(status, "IUtil", cloopVTable->version, 4); + StatusType::checkException(status); + return 0; + } + StatusType::clearException(status); + IInt128* ret = static_cast(this->cloopVTable)->getInt128(this, status); + StatusType::checkException(status); + return ret; + } }; class IOffsetsCallback : public IVersioned @@ -5856,6 +5872,45 @@ namespace Firebird } }; + class IInt128 : public IVersioned + { + public: + struct VTable : public IVersioned::VTable + { + void (CLOOP_CARG *toString)(IInt128* self, IStatus* status, const FB_I128* from, int scale, unsigned bufferLength, char* buffer) throw(); + void (CLOOP_CARG *fromString)(IInt128* self, IStatus* status, int scale, const char* from, FB_I128* to) throw(); + }; + + protected: + IInt128(DoNotInherit) + : IVersioned(DoNotInherit()) + { + } + + ~IInt128() + { + } + + public: + static const unsigned VERSION = 2; + + static const unsigned STRING_SIZE = 46; + + template void toString(StatusType* status, const FB_I128* from, int scale, unsigned bufferLength, char* buffer) + { + StatusType::clearException(status); + static_cast(this->cloopVTable)->toString(this, status, from, scale, bufferLength, buffer); + StatusType::checkException(status); + } + + template void fromString(StatusType* status, int scale, const char* from, FB_I128* to) + { + StatusType::clearException(status); + static_cast(this->cloopVTable)->fromString(this, status, scale, from, to); + StatusType::checkException(status); + } + }; + class IReplicatedRecord : public IVersioned { public: @@ -14332,6 +14387,7 @@ namespace Firebird this->decodeTimeStampTz = &Name::cloopdecodeTimeStampTzDispatcher; this->encodeTimeTz = &Name::cloopencodeTimeTzDispatcher; this->encodeTimeStampTz = &Name::cloopencodeTimeStampTzDispatcher; + this->getInt128 = &Name::cloopgetInt128Dispatcher; } } vTable; @@ -14630,6 +14686,21 @@ namespace Firebird StatusType::catchException(&status2); } } + + static IInt128* CLOOP_CARG cloopgetInt128Dispatcher(IUtil* self, IStatus* status) throw() + { + StatusType status2(status); + + try + { + return static_cast(self)->Name::getInt128(&status2); + } + catch (...) + { + StatusType::catchException(&status2); + return static_cast(0); + } + } }; template > > @@ -14666,6 +14737,7 @@ namespace Firebird virtual void decodeTimeStampTz(StatusType* status, const ISC_TIMESTAMP_TZ* timeStampTz, unsigned* year, unsigned* month, unsigned* day, unsigned* hours, unsigned* minutes, unsigned* seconds, unsigned* fractions, unsigned timeZoneBufferLength, char* timeZoneBuffer) = 0; virtual void encodeTimeTz(StatusType* status, ISC_TIME_TZ* timeTz, unsigned hours, unsigned minutes, unsigned seconds, unsigned fractions, const char* timeZone) = 0; virtual void encodeTimeStampTz(StatusType* status, ISC_TIMESTAMP_TZ* timeStampTz, unsigned year, unsigned month, unsigned day, unsigned hours, unsigned minutes, unsigned seconds, unsigned fractions, const char* timeZone) = 0; + virtual IInt128* getInt128(StatusType* status) = 0; }; template @@ -18081,6 +18153,73 @@ namespace Firebird virtual void fromString(StatusType* status, const char* from, FB_DEC34* to) = 0; }; + template + class IInt128BaseImpl : public Base + { + public: + typedef IInt128 Declaration; + + IInt128BaseImpl(DoNotInherit = DoNotInherit()) + { + static struct VTableImpl : Base::VTable + { + VTableImpl() + { + this->version = Base::VERSION; + this->toString = &Name::clooptoStringDispatcher; + this->fromString = &Name::cloopfromStringDispatcher; + } + } vTable; + + this->cloopVTable = &vTable; + } + + static void CLOOP_CARG clooptoStringDispatcher(IInt128* self, IStatus* status, const FB_I128* from, int scale, unsigned bufferLength, char* buffer) throw() + { + StatusType status2(status); + + try + { + static_cast(self)->Name::toString(&status2, from, scale, bufferLength, buffer); + } + catch (...) + { + StatusType::catchException(&status2); + } + } + + static void CLOOP_CARG cloopfromStringDispatcher(IInt128* self, IStatus* status, int scale, const char* from, FB_I128* to) throw() + { + StatusType status2(status); + + try + { + static_cast(self)->Name::fromString(&status2, scale, from, to); + } + catch (...) + { + StatusType::catchException(&status2); + } + } + }; + + template > > + class IInt128Impl : public IInt128BaseImpl + { + protected: + IInt128Impl(DoNotInherit = DoNotInherit()) + { + } + + public: + virtual ~IInt128Impl() + { + } + + virtual void toString(StatusType* status, const FB_I128* from, int scale, unsigned bufferLength, char* buffer) = 0; + virtual void fromString(StatusType* status, int scale, const char* from, FB_I128* to) = 0; + }; + template class IReplicatedRecordBaseImpl : public Base { diff --git a/src/include/firebird/impl/blr.h b/src/include/firebird/impl/blr.h index a524012f3b..70a41ac159 100644 --- a/src/include/firebird/impl/blr.h +++ b/src/include/firebird/impl/blr.h @@ -69,7 +69,7 @@ #define blr_bool (unsigned char)23 #define blr_dec64 (unsigned char)24 #define blr_dec128 (unsigned char)25 -#define blr_dec_fixed (unsigned char)26 +#define blr_int128 (unsigned char)26 #define blr_sql_time_tz (unsigned char)28 #define blr_timestamp_tz (unsigned char)29 diff --git a/src/include/firebird/impl/consts_pub.h b/src/include/firebird/impl/consts_pub.h index 6252c49775..9f34fc29f6 100644 --- a/src/include/firebird/impl/consts_pub.h +++ b/src/include/firebird/impl/consts_pub.h @@ -129,6 +129,7 @@ #define isc_dpb_decfloat_bind 94 #define isc_dpb_decfloat_round 95 #define isc_dpb_decfloat_traps 96 +#define isc_dpb_int128_bind 97 /**************************************************/ diff --git a/src/include/firebird/impl/dsc_pub.h b/src/include/firebird/impl/dsc_pub.h index 4429673cce..bb1f3e9d12 100644 --- a/src/include/firebird/impl/dsc_pub.h +++ b/src/include/firebird/impl/dsc_pub.h @@ -63,7 +63,7 @@ #define dtype_boolean 21 #define dtype_dec64 22 #define dtype_dec128 23 -#define dtype_dec_fixed 24 +#define dtype_int128 24 #define dtype_sql_time_tz 25 #define dtype_timestamp_tz 26 #define DTYPE_TYPE_MAX 27 diff --git a/src/include/firebird/impl/sqlda_pub.h b/src/include/firebird/impl/sqlda_pub.h index 5b3f95b863..2e64e6024a 100644 --- a/src/include/firebird/impl/sqlda_pub.h +++ b/src/include/firebird/impl/sqlda_pub.h @@ -78,9 +78,9 @@ typedef struct #define SQL_TYPE_TIME 560 #define SQL_TYPE_DATE 570 #define SQL_INT64 580 +#define SQL_INT128 32752 #define SQL_TIMESTAMP_TZ 32754 #define SQL_TIME_TZ 32756 -#define SQL_DEC_FIXED 32758 #define SQL_DEC16 32760 #define SQL_DEC34 32762 #define SQL_BOOLEAN 32764 diff --git a/src/include/firebird/impl/types_pub.h b/src/include/firebird/impl/types_pub.h index 5fcc880dc2..bcd8653922 100644 --- a/src/include/firebird/impl/types_pub.h +++ b/src/include/firebird/impl/types_pub.h @@ -197,7 +197,12 @@ struct FB_DEC34_t { ISC_UINT64 fb_data[2]; }; +struct FB_I128_t { + ISC_UINT64 fb_data[2]; +}; + typedef struct FB_DEC16_t FB_DEC16; typedef struct FB_DEC34_t FB_DEC34; +typedef struct FB_I128_t FB_I128; #endif /* FIREBIRD_IMPL_TYPES_PUB_H */ diff --git a/src/isql/isql.epp b/src/isql/isql.epp index 733f6c9133..5a73a6a30f 100644 --- a/src/isql/isql.epp +++ b/src/isql/isql.epp @@ -320,6 +320,7 @@ IsqlGlobals::IsqlGlobals() df16 = Firebird::UtilInterfacePtr()->getDecFloat16(&statusWrapper); df34 = Firebird::UtilInterfacePtr()->getDecFloat34(&statusWrapper); + i128 = Firebird::UtilInterfacePtr()->getInt128(&statusWrapper); } // I s q l G l o b a l s : : p r i n t f @@ -1838,7 +1839,7 @@ bool ISQL_printNumericType(const char* fieldName, const int fieldType, const int case INTEGER: case BIGINT: case DOUBLE_PRECISION: - case DEC_FIXED_TYPE: + case blr_int128: // Handle Integral subtypes NUMERIC and DECIMAL // We are ODS >= 10 and could be any Dialect @@ -1872,8 +1873,8 @@ bool ISQL_printNumericType(const char* fieldName, const int fieldType, const int case DOUBLE_PRECISION: isqlGlob.printf("NUMERIC(15, %d)", -fieldScale); break; - case DEC_FIXED_TYPE: - isqlGlob.printf("NUMERIC(34, %d)", -fieldScale); + case blr_int128: + isqlGlob.printf("NUMERIC(38, %d)", -fieldScale); break; } } @@ -2288,6 +2289,7 @@ static processing_state add_row(TEXT* tabname) double* dvalue; FB_DEC16* d64value; FB_DEC34* d128value; + FB_I128* i128value; UCHAR* boolean; ISC_QUAD* blobid; vary* avary; @@ -2361,11 +2363,27 @@ static processing_state add_row(TEXT* tabname) if ((!isqlGlob.df16) || (fbStatus->getState() & Firebird::IStatus::STATE_ERRORS)) { STDERROUT("Input parsing problem"); + ISQL_errmsg(fbStatus); + done = true; + } + break; + + case SQL_INT128: + scale = msg->getScale(fbStatus, i); + if (ISQL_errmsg(fbStatus)) + return (SKIP); + + i128value = (FB_I128*) datap; + if (isqlGlob.i128) + isqlGlob.i128->fromString(fbStatus, scale, lastInputLine, i128value); + if ((!isqlGlob.i128) || (fbStatus->getState() & Firebird::IStatus::STATE_ERRORS)) + { + STDERROUT("Input parsing problem"); + ISQL_errmsg(fbStatus); done = true; } break; - case SQL_DEC_FIXED: case SQL_DEC34: d128value = (FB_DEC34*) datap; if (isqlGlob.df34) @@ -2373,6 +2391,7 @@ static processing_state add_row(TEXT* tabname) if ((!isqlGlob.df34) || (fbStatus->getState() & Firebird::IStatus::STATE_ERRORS)) { STDERROUT("Input parsing problem"); + ISQL_errmsg(fbStatus); done = true; } break; @@ -3068,6 +3087,7 @@ static processing_state bulk_insert_hack(const char* command) tm times; FB_DEC16* d64value; FB_DEC34* d128value; + FB_I128* i128value; // Initialize the time structure. memset(×, 0, sizeof(times)); char msec_str[5] = ""; @@ -3134,18 +3154,38 @@ static processing_state bulk_insert_hack(const char* command) if ((!isqlGlob.df16) || (fbStatus->getState() & Firebird::IStatus::STATE_ERRORS)) { STDERROUT("Input parsing problem"); + ISQL_errmsg(fbStatus); done = true; } break; case SQL_DEC34: - case SQL_DEC_FIXED: d128value = (FB_DEC34*) datap; if (isqlGlob.df34) isqlGlob.df34->fromString(fbStatus, get_numeric_value(lastPos).c_str(), d128value); if ((!isqlGlob.df34) || (fbStatus->getState() & Firebird::IStatus::STATE_ERRORS)) { STDERROUT("Input parsing problem"); + ISQL_errmsg(fbStatus); + done = true; + } + break; + + case SQL_INT128: + scale = message->getScale(fbStatus, i); + if (ISQL_errmsg(fbStatus)) + return (SKIP); + + i128value = (FB_I128*) datap; + if (isqlGlob.i128) + { + isqlGlob.i128->fromString(fbStatus, scale, + get_numeric_value(lastPos).c_str(), i128value); + } + if ((!isqlGlob.i128) || (fbStatus->getState() & Firebird::IStatus::STATE_ERRORS)) + { + STDERROUT("Input parsing problem"); + ISQL_errmsg(fbStatus); done = true; } break; @@ -7312,7 +7352,6 @@ static unsigned print_item(TEXT** s, const IsqlVar* var, const unsigned length) break; case SQL_DEC34: - case SQL_DEC_FIXED: { char decStr[Firebird::IDecFloat34::STRING_SIZE]; if (isqlGlob.df34) @@ -7325,9 +7364,34 @@ static unsigned print_item(TEXT** s, const IsqlVar* var, const unsigned length) strcpy(decStr, convErr); if (setValues.List) - isqlGlob.printf("%*.*s%s", sizeof(decStr) - 1, sizeof(decStr) - 1, decStr, NEWLINE); + { + isqlGlob.printf("%*.*s%s", int(sizeof(decStr) - 1), int(sizeof(decStr) - 1), + decStr, NEWLINE); + } else - sprintf(p, "%*.*s ", sizeof(decStr) - 1, sizeof(decStr) - 1, decStr); + sprintf(p, "%*.*s ", int(sizeof(decStr) - 1), int(sizeof(decStr) - 1), decStr); + } + break; + + case SQL_INT128: + { + char intStr[Firebird::IInt128::STRING_SIZE]; + if (isqlGlob.i128) + { + isqlGlob.i128->toString(fbStatus, var->value.asInt128, dscale, sizeof(intStr), intStr); + if (ISQL_errmsg(fbStatus)) + strcpy(intStr, convErr); + } + else + strcpy(intStr, convErr); + + if (setValues.List) + { + isqlGlob.printf("%*.*s%s", int(sizeof(intStr) - 1), int(sizeof(intStr) - 1), + intStr, NEWLINE); + } + else + sprintf(p, "%*.*s ", int(sizeof(intStr) - 1), int(sizeof(intStr) - 1), intStr); } break; @@ -8169,7 +8233,9 @@ static unsigned process_message_display(Firebird::IMessageMetadata* message, uns case SQL_DEC16: disp_length = Firebird::IDecFloat16::STRING_SIZE - 1; break; - case SQL_DEC_FIXED: + case SQL_INT128: + disp_length = Firebird::IInt128::STRING_SIZE - 1; + break; case SQL_DEC34: disp_length = Firebird::IDecFloat34::STRING_SIZE - 1; break; @@ -8847,8 +8913,8 @@ static const char* sqltype_to_string(unsigned sqltype) return "DECFLOAT(16)"; case SQL_DEC34: return "DECFLOAT(34)"; - case SQL_DEC_FIXED: - return "DECIMAL FIXED"; + case SQL_INT128: + return "NUMERIC(38)"; case SQL_D_FLOAT: return "D_FLOAT"; case SQL_TIMESTAMP: diff --git a/src/isql/isql.h b/src/isql/isql.h index 17e15049a1..ea239e6ecd 100644 --- a/src/isql/isql.h +++ b/src/isql/isql.h @@ -306,7 +306,6 @@ const int BIGINT = 16; const int BOOLEAN_TYPE = 23; const int DEC64_TYPE = 24; const int DEC128_TYPE = 25; -const int DEC_FIXED_TYPE = 26; static const sqltypes Column_types[] = { {SMALLINT, "SMALLINT"}, // keyword @@ -326,7 +325,7 @@ static const sqltypes Column_types[] = { {BOOLEAN_TYPE, "BOOLEAN"}, // keyword {DEC64_TYPE, "DECFLOAT(16)"}, {DEC128_TYPE, "DECFLOAT(34)"}, - {DEC_FIXED_TYPE, ""}, + {blr_int128, "INT64"}, {blr_sql_time_tz, "TIME WITH TIME ZONE"}, // keyword {blr_timestamp_tz, "TIMESTAMP WITH TIME ZONE"}, // keyword {0, ""} @@ -417,6 +416,7 @@ public: USHORT att_charset; Firebird::IDecFloat16* df16; Firebird::IDecFloat34* df34; + Firebird::IInt128* i128; void printf(const char* buffer, ...); void prints(const char* buffer); @@ -478,6 +478,7 @@ struct IsqlVar char* asChar; FB_DEC16* asDec16; FB_DEC34* asDec34; + FB_I128* asInt128; void* setPtr; }; TypeMix value; diff --git a/src/jrd/Attachment.cpp b/src/jrd/Attachment.cpp index f23b92f11c..798a69cc41 100644 --- a/src/jrd/Attachment.cpp +++ b/src/jrd/Attachment.cpp @@ -243,7 +243,7 @@ Jrd::Attachment::Attachment(MemoryPool* pool, Database* dbb, const InitialOption att_internal(*pool), att_dyn_req(*pool), att_dec_status(DecimalStatus::DEFAULT), - att_dec_binding(DecimalBinding::DEFAULT), + att_dec_binding(NumericBinding::DEFAULT), att_charsets(*pool), att_charset_ids(*pool), att_pools(*pool), diff --git a/src/jrd/Attachment.h b/src/jrd/Attachment.h index f92420890d..b4c47f434e 100644 --- a/src/jrd/Attachment.h +++ b/src/jrd/Attachment.h @@ -363,8 +363,11 @@ public: void resetAttachment(Attachment* attachment) const; private: + void setBinding(Firebird::string option, Firebird::NumericBinding& bind); + Firebird::DecimalStatus decFloatStatus = Firebird::DecimalStatus::DEFAULT; - Firebird::DecimalBinding decFloatBinding = Firebird::DecimalBinding::DEFAULT; + Firebird::NumericBinding decFloatBinding = Firebird::NumericBinding::DEFAULT; + Firebird::NumericBinding int128Binding = Firebird::NumericBinding::DEFAULT; Firebird::TimeZoneUtil::Bind timeZoneBind = Firebird::TimeZoneUtil::BIND_NATIVE; USHORT originalTimeZone = Firebird::TimeZoneUtil::GMT_ZONE; @@ -466,7 +469,8 @@ public: Firebird::Array att_dyn_req; // internal dyn statements Firebird::ICryptKeyCallback* att_crypt_callback; // callback for DB crypt Firebird::DecimalStatus att_dec_status; // error handling and rounding - Firebird::DecimalBinding att_dec_binding; // use legacy datatype for DecFloat in outer world + Firebird::NumericBinding att_dec_binding; // use legacy datatype for DecFloat in outer world + Firebird::NumericBinding att_i128_binding; // use legacy datatype for INT128 in outer world jrd_req* findSystemRequest(thread_db* tdbb, USHORT id, USHORT which); diff --git a/src/jrd/ExtEngineManager.cpp b/src/jrd/ExtEngineManager.cpp index cbf5e69cb3..fcea4ec0ef 100644 --- a/src/jrd/ExtEngineManager.cpp +++ b/src/jrd/ExtEngineManager.cpp @@ -1570,9 +1570,6 @@ void ExtEngineManager::makeTrigger(thread_db* tdbb, CompilerScratch* csb, Jrd::T if (field) { dsc d(relFormat->fmt_desc[i]); - if (d.dsc_dtype == dtype_dec_fixed) - d.dsc_dtype = dtype_dec128; - fieldsMsg->addItem(field->fld_name, !field->fld_not_null, d); } } diff --git a/src/jrd/PreparedStatement.cpp b/src/jrd/PreparedStatement.cpp index 4bcf02b7c7..fb98f5bf30 100644 --- a/src/jrd/PreparedStatement.cpp +++ b/src/jrd/PreparedStatement.cpp @@ -109,10 +109,10 @@ namespace item.length = sizeof(Decimal128); break; - case dtype_dec_fixed: - item.type = SQL_DEC_FIXED; + case dtype_int128: + item.type = SQL_INT128; item.scale = desc->dsc_scale; - item.length = sizeof(DecimalFixed); + item.length = sizeof(Int128); break; case dtype_sql_date: diff --git a/src/jrd/Routine.cpp b/src/jrd/Routine.cpp index bbefc73e88..2a88bd4f9e 100644 --- a/src/jrd/Routine.cpp +++ b/src/jrd/Routine.cpp @@ -44,7 +44,7 @@ MsgMetadata* Routine::createMetadata(const Array >& paramet ++i) { dsc d((*i)->prm_desc); - if (isExtern && d.dsc_dtype == dtype_dec_fixed) + if (isExtern && d.dsc_dtype == dtype_int128) d.dsc_dtype = dtype_dec128; metadata->addItem((*i)->prm_name, (*i)->prm_nullable, d); } diff --git a/src/jrd/align.h b/src/jrd/align.h index e8be9e6468..c7acb0331d 100644 --- a/src/jrd/align.h +++ b/src/jrd/align.h @@ -33,6 +33,7 @@ Maximum alignments for corresponding data types are defined in dsc.h */ #include "../common/DecFloat.h" +#include "../common/Int128.h" #include "firebird/impl/blr.h" /* The following macro must be defined as the highest-numericly-valued @@ -70,7 +71,7 @@ static const USHORT gds_cvt_blr_dtype[DTYPE_BLR_MAX + 1] = dtype_boolean, // blr_bool == 23 dtype_dec64, /* blr_dec64 == 24 */ dtype_dec128, /* blr_dec128 == 25 */ - dtype_dec_fixed, /* blr_dec_fixed == 26 */ + dtype_int128, /* blr_int128 == 26 */ dtype_double, /* blr_double == 27 */ dtype_sql_time_tz, /* blr_sql_time_tz == 28 */ dtype_timestamp_tz, /* blr_timestamp_tz == 29 */ @@ -111,7 +112,7 @@ static const USHORT type_alignments[DTYPE_TYPE_MAX] = sizeof(UCHAR), /* dtype_boolean */ sizeof(Firebird::Decimal64),/* dtype_dec64 */ sizeof(Firebird::Decimal64),/* dtype_dec128 */ - sizeof(Firebird::Decimal64),/* dtype_dec_fixed */ + sizeof(Firebird::Decimal64),/* dtype_int128 */ sizeof(GDS_TIME), /* dtype_sql_time_tz */ sizeof(GDS_DATE) /* dtype_timestamp_tz */ }; @@ -142,7 +143,7 @@ static const USHORT type_lengths[DTYPE_TYPE_MAX] = sizeof(UCHAR), /* dtype_boolean */ sizeof(Firebird::Decimal64),/* dtype_dec64 */ sizeof(Firebird::Decimal128),/*dtype_dec128 */ - sizeof(Firebird::DecimalFixed), /* dtype_dec_fixed */ + sizeof(Firebird::Int128), /* dtype_int128 */ sizeof(ISC_TIME_TZ), /* dtype_sql_time_tz */ sizeof(ISC_TIMESTAMP_TZ) /* dtype_timestamp_tz */ }; @@ -176,7 +177,7 @@ static const USHORT type_significant_bits[DTYPE_TYPE_MAX] = 0, // dtype_boolean 0, // dtype_dec64 0, // dtype_dec128 - 0, // dtype_dec_fixed + 0, // dtype_int128 0, // dtype_sql_time_tz 0 // dtype_timestamp_tz }; diff --git a/src/jrd/cvt.cpp b/src/jrd/cvt.cpp index 0a9d2031ef..6ccd886402 100644 --- a/src/jrd/cvt.cpp +++ b/src/jrd/cvt.cpp @@ -223,19 +223,20 @@ UCHAR CVT_get_numeric(const UCHAR* string, const USHORT length, SSHORT* scale, v // tricky: the value doesn't always become negative after an // overflow! - if (value >= NUMERIC_LIMIT) + if (!over) { - // possibility of an overflow - if ((value > NUMERIC_LIMIT) || (*p > '8' && sign == -1) || (*p > '7' && sign != -1)) + if (value >= NUMERIC_LIMIT) { - over = true; - break; + // possibility of an overflow + if ((value > NUMERIC_LIMIT) || (*p > '8' && sign == -1) || (*p > '7' && sign != -1)) + over = true; } + + // Force the subtraction to be performed before the addition, + // thus preventing a possible signed arithmetic overflow. + value = value * 10 + (*p - '0'); } - // Force the subtraction to be performed before the addition, - // thus preventing a possible signed arithmetic overflow. - value = value * 10 + (*p - '0'); if (fraction) --local_scale; } @@ -262,6 +263,8 @@ UCHAR CVT_get_numeric(const UCHAR* string, const USHORT length, SSHORT* scale, v if ((local_scale > MAX_SCHAR) || (local_scale < MIN_SCHAR)) over = true; + *scale = local_scale; + if ((!over) && ((p < end) || // there is an exponent ((value < 0) && (sign != -1)))) // MAX_SINT64+1 wrapped around { @@ -278,8 +281,6 @@ UCHAR CVT_get_numeric(const UCHAR* string, const USHORT length, SSHORT* scale, v return dtype_dec128; } - *scale = local_scale; - // The literal has already been converted to a 64-bit integer: return // a long if the value fits into a long, else return an int64. diff --git a/src/jrd/cvt2.cpp b/src/jrd/cvt2.cpp index 96f2e24260..8959088287 100644 --- a/src/jrd/cvt2.cpp +++ b/src/jrd/cvt2.cpp @@ -85,7 +85,7 @@ const BYTE CVT2_compare_priority[] = 9, // dtype_long // Move quad up by one to make room for int64 at its proper place in the table. 11, // dtype_quad - // Also leave space for dec_fixed, dec64 and dec 128. + // Also leave space for int128, dec64 and dec 128. 15, // dtype_real 16, // dtype_double 17, // dtype_d_float @@ -99,8 +99,8 @@ const BYTE CVT2_compare_priority[] = 10, // dtype_int64 - goes right after long 25, // dtype_dbkey - compares with nothing except itself 26, // dtype_boolean - compares with nothing except itself - 12, // dtype_dec_fixed - go after quad - 13, // dec64 - go after dtype_dec_fixed + 12, // dtype_int128 - go after quad + 13, // dec64 - go after dtype_int128 14, // dec128 - go after dec64 and before real 20, // dtype_sql_time_tz - go after dtype_sql_time 22 // dtype_timestamp_tz - go after dtype_timestamp @@ -305,8 +305,8 @@ int CVT2_compare(const dsc* arg1, const dsc* arg2, Firebird::DecimalStatus decSt case dtype_dec128: return ((Decimal128*) p1)->compare(decSt, *(Decimal128*) p2); - case dtype_dec_fixed: - return ((DecimalFixed*) p1)->compare(decSt, *(DecimalFixed*) p2); + case dtype_int128: + return ((Int128*) p1)->compare(*(Int128*) p2); case dtype_boolean: return *p1 == *p2 ? 0 : *p1 < *p2 ? -1 : 1; @@ -564,7 +564,7 @@ int CVT2_compare(const dsc* arg1, const dsc* arg2, Firebird::DecimalStatus decSt return temp1.compare(decSt, temp2); } - case dtype_dec_fixed: + case dtype_int128: { SSHORT scale; if (arg2->dsc_dtype > dtype_varying) @@ -572,9 +572,9 @@ int CVT2_compare(const dsc* arg1, const dsc* arg2, Firebird::DecimalStatus decSt else scale = arg1->dsc_scale; - const DecimalFixed temp1 = CVT_get_dec_fixed(arg1, scale, decSt, ERR_post); - const DecimalFixed temp2 = CVT_get_dec_fixed(arg2, scale, decSt, ERR_post); - return temp1.compare(decSt, temp2); + const Int128 temp1 = CVT_get_int128(arg1, scale, decSt, ERR_post); + const Int128 temp2 = CVT_get_int128(arg2, scale, decSt, ERR_post); + return temp1.compare(temp2); } case dtype_blob: diff --git a/src/jrd/dfw.epp b/src/jrd/dfw.epp index 367c492aea..ce4c7dced0 100644 --- a/src/jrd/dfw.epp +++ b/src/jrd/dfw.epp @@ -1302,7 +1302,7 @@ USHORT DFW_assign_index_type(thread_db* tdbb, const Firebird::MetaName& name, SS return idx_boolean; case dtype_dec64: case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: return idx_decimal; default: return idx_numeric; diff --git a/src/jrd/evl.cpp b/src/jrd/evl.cpp index 64ea89226b..a25cae27a3 100644 --- a/src/jrd/evl.cpp +++ b/src/jrd/evl.cpp @@ -432,8 +432,8 @@ void EVL_make_value(thread_db* tdbb, const dsc* desc, impure_value* value, Memor value->vlu_misc.vlu_dec128 = *((Decimal128*) from.dsc_address); return; - case dtype_dec_fixed: - value->vlu_misc.vlu_dec_fixed = *((DecimalFixed*) from.dsc_address); + case dtype_int128: + value->vlu_misc.vlu_int128 = *((Int128*) from.dsc_address); return; case dtype_sql_time: diff --git a/src/jrd/exe.h b/src/jrd/exe.h index 2c6514486e..330545f533 100644 --- a/src/jrd/exe.h +++ b/src/jrd/exe.h @@ -64,7 +64,6 @@ DEFINE_TRACE_ROUTINE(cmp_trace); #endif class VaryingString; -struct dsc; namespace Jrd { @@ -463,7 +462,7 @@ public: csb_currentForNode(NULL), csb_currentDMLNode(NULL), csb_currentAssignTarget(NULL), - csb_preferredDataType(0), + csb_preferredDesc(NULL), csb_rpt(p) { csb_dbg_info = FB_NEW_POOL(p) Firebird::DbgInfo(p); @@ -546,7 +545,7 @@ public: ForNode* csb_currentForNode; StmtNode* csb_currentDMLNode; // could be StoreNode or ModifyNode ExprNode* csb_currentAssignTarget; - UCHAR csb_preferredDataType; // expected by receiving side datatype + dsc* csb_preferredDesc; // expected by receiving side data format struct csb_repeat { diff --git a/src/jrd/fun.epp b/src/jrd/fun.epp index c5aee2f1fc..e1a9ae1cc7 100644 --- a/src/jrd/fun.epp +++ b/src/jrd/fun.epp @@ -606,7 +606,7 @@ void FUN_evaluate(thread_db* tdbb, const Function* function, const NestValueArra break; case dtype_dec128: - case dtype_dec_fixed: + case dtype_int128: { const Decimal128 d = MOV_get_dec128(tdbb, input); if (parameter->prm_fun_mechanism == FUN_value) @@ -820,21 +820,6 @@ void FUN_evaluate(thread_db* tdbb, const Function* function, const NestValueArra Arg::Str(function->getName().toString())); } break; - - case dtype_dec_fixed: - if (value->vlu_misc.vlu_dec_fixed.isInf()) - { - status_exception::raise(Arg::Gds(isc_expression_eval_err) << - Arg::Gds(isc_udf_fp_overflow) << - Arg::Str(function->getName().toString())); - } - else if (value->vlu_misc.vlu_dec_fixed.isNan()) - { - status_exception::raise(Arg::Gds(isc_expression_eval_err) << - Arg::Gds(isc_udf_fp_nan) << - Arg::Str(function->getName().toString())); - } - break; } request->req_flags &= ~req_null; @@ -1148,8 +1133,8 @@ static void invoke(thread_db* tdbb, value->vlu_misc.vlu_dec128 = CALL_UDF(tdbb, function->fun_entrypoint, args); break; - case dtype_dec_fixed: - value->vlu_misc.vlu_dec_fixed = CALL_UDF(tdbb, function->fun_entrypoint, args); + case dtype_int128: + value->vlu_misc.vlu_int128 = CALL_UDF(tdbb, function->fun_entrypoint, args); break; case dtype_timestamp: diff --git a/src/jrd/jrd.cpp b/src/jrd/jrd.cpp index 08fb2f2330..afb052bbba 100644 --- a/src/jrd/jrd.cpp +++ b/src/jrd/jrd.cpp @@ -1020,6 +1020,7 @@ namespace Jrd string dpb_decfloat_bind; string dpb_decfloat_round; string dpb_decfloat_traps; + string dpb_int128_bind; public: static const ULONG DPB_FLAGS_MASK = DBB_damaged; @@ -1062,6 +1063,43 @@ namespace Jrd } }; + void Attachment::InitialOptions::setBinding(string option, NumericBinding& bind) + { + option.lower(); + + if (option == "native") + bind = NumericBinding::DEFAULT; + else if (option == "char" || option == "character") + bind = NumericBinding(NumericBinding::NUM_TEXT); + else if (option == "double" || option == "double precision") + bind = NumericBinding(NumericBinding::NUM_DOUBLE); + else if (option == "bigint") + bind = NumericBinding(NumericBinding::NUM_INT64); + else if (option.substr(0, 7) == "bigint,") + { + const char* p = option.c_str() + 7; + + while (*p == ' ') + ++p; + + const char* start = p; + int scale = 0; + + while (*p >= '0' && *p <= '9') + { + scale = scale * 10 + (*p - '0'); + ++p; + } + + if (*p != '\0' || p - start == 0 || p - start > 2 || scale > NumericBinding::MAX_SCALE) + (Arg::Gds(isc_invalid_decfloat_bind) << option).raise(); + + bind = NumericBinding(NumericBinding::NUM_INT64, static_cast(-scale)); + } + else + (Arg::Gds(isc_invalid_decfloat_bind) << option).raise(); + } + Attachment::InitialOptions::InitialOptions(const DatabaseOptions& options) { if (options.dpb_time_zone_bind.hasData()) @@ -1078,42 +1116,10 @@ namespace Jrd } if (options.dpb_decfloat_bind.hasData()) - { - auto option = options.dpb_decfloat_bind; - option.lower(); + setBinding(options.dpb_decfloat_bind, decFloatBinding); - if (option == "native") - decFloatBinding = DecimalBinding::DEFAULT; - else if (option == "char" || option == "character") - decFloatBinding = DecimalBinding(DecimalBinding::DEC_TEXT); - else if (option == "double" || option == "double precision") - decFloatBinding = DecimalBinding(DecimalBinding::DEC_DOUBLE); - else if (option == "bigint") - decFloatBinding = DecimalBinding(DecimalBinding::DEC_NUMERIC); - else if (option.substr(0, 7) == "bigint,") - { - const char* p = option.c_str() + 7; - - while (*p == ' ') - ++p; - - const char* start = p; - int scale = 0; - - while (*p >= '0' && *p <= '9') - { - scale = scale * 10 + (*p - '0'); - ++p; - } - - if (*p != '\0' || p - start == 0 || p - start > 2 || scale > DecimalBinding::MAX_SCALE) - (Arg::Gds(isc_invalid_decfloat_bind) << option).raise(); - - decFloatBinding = DecimalBinding(DecimalBinding::DEC_NUMERIC, static_cast(-scale)); - } - else - (Arg::Gds(isc_invalid_decfloat_bind) << option).raise(); - } + if (options.dpb_int128_bind.hasData()) + setBinding(options.dpb_int128_bind, int128Binding); if (options.dpb_decfloat_round.hasData()) { @@ -1172,6 +1178,7 @@ namespace Jrd // reset DecFloat options attachment->att_dec_status = decFloatStatus; attachment->att_dec_binding = decFloatBinding; + attachment->att_i128_binding = int128Binding; // reset time zone options attachment->att_timezone_bind = timeZoneBind; @@ -6969,6 +6976,10 @@ void DatabaseOptions::get(const UCHAR* dpb, USHORT dpb_length, bool& invalid_cli rdr.getString(dpb_decfloat_bind); break; + case isc_dpb_int128_bind: + rdr.getString(dpb_int128_bind); + break; + case isc_dpb_decfloat_round: rdr.getString(dpb_decfloat_round); break; diff --git a/src/jrd/mov.cpp b/src/jrd/mov.cpp index ad15676f28..9c10a83603 100644 --- a/src/jrd/mov.cpp +++ b/src/jrd/mov.cpp @@ -450,42 +450,6 @@ void MOV_move(Jrd::thread_db* tdbb, /*const*/ dsc* from, dsc* to) } -void MOV_move_ext(Jrd::thread_db* tdbb, /*const*/ dsc* from, dsc* to, bool toExtern) -{ -/************************************** - * - * M O V _ m o v e _ e x t - * - ************************************** - * - * Functional description - * Move data to/from outer world. - * - **************************************/ - - MOV_move(tdbb, from, to); - - switch (to->dsc_dtype) - { - case dtype_dec_fixed: - if (toExtern) - { - ((Decimal128*) to->dsc_address)->setScale(tdbb->getAttachment()->att_dec_status, - to->dsc_scale); - } - else - { - ((DecimalFixed*) to->dsc_address)->exactInt(tdbb->getAttachment()->att_dec_status, - to->dsc_scale); - } - break; - - default: - break; - } -} - - Decimal64 MOV_get_dec64(Jrd::thread_db* tdbb, const dsc* desc) { /************************************** @@ -510,7 +474,7 @@ Decimal128 MOV_get_dec128(Jrd::thread_db* tdbb, const dsc* desc) } -DecimalFixed MOV_get_dec_fixed(Jrd::thread_db* tdbb, const dsc* desc, SSHORT scale) +Int128 MOV_get_int128(Jrd::thread_db* tdbb, const dsc* desc, SSHORT scale) { /************************************** * @@ -518,7 +482,7 @@ DecimalFixed MOV_get_dec_fixed(Jrd::thread_db* tdbb, const dsc* desc, SSHORT sca * **************************************/ - return CVT_get_dec_fixed(desc, scale, tdbb->getAttachment()->att_dec_status, ERR_post); + return CVT_get_int128(desc, scale, tdbb->getAttachment()->att_dec_status, ERR_post); } diff --git a/src/jrd/mov_proto.h b/src/jrd/mov_proto.h index 31f59d458c..4e600889ab 100644 --- a/src/jrd/mov_proto.h +++ b/src/jrd/mov_proto.h @@ -52,10 +52,9 @@ int MOV_make_string2(Jrd::thread_db*, const dsc*, USHORT, UCHAR**, Jrd::MoveBuf Firebird::string MOV_make_string2(Jrd::thread_db* tdbb, const dsc* desc, USHORT ttype, bool limit = true); void MOV_move(Jrd::thread_db*, /*const*/ dsc*, dsc*); -void MOV_move_ext(Jrd::thread_db* tdbb, /*const*/ dsc* from, dsc* to, bool toExtern); Firebird::Decimal64 MOV_get_dec64(Jrd::thread_db*, const dsc*); Firebird::Decimal128 MOV_get_dec128(Jrd::thread_db*, const dsc*); -Firebird::DecimalFixed MOV_get_dec_fixed(Jrd::thread_db*, const dsc*, SSHORT); +Firebird::Int128 MOV_get_int128(Jrd::thread_db*, const dsc*, SSHORT); namespace Jrd { diff --git a/src/jrd/opt.cpp b/src/jrd/opt.cpp index ec23786b71..7c5866bbbf 100644 --- a/src/jrd/opt.cpp +++ b/src/jrd/opt.cpp @@ -419,7 +419,7 @@ static const UCHAR sort_dtypes[] = SKD_bytes, // dtype_boolean SKD_dec64, // dtype_dec64 SKD_dec128, // dtype_dec128 - SKD_dec128, // dtype_dec_fixed + SKD_int128, // dtype_int128 SKD_sql_time_tz, // dtype_sql_time_tz SKD_timestamp_tz // dtype_timestamp_tz }; diff --git a/src/jrd/par.cpp b/src/jrd/par.cpp index c4dada62e8..a0eabbe1bf 100644 --- a/src/jrd/par.cpp +++ b/src/jrd/par.cpp @@ -405,9 +405,9 @@ USHORT PAR_datatype(BlrReader& blrReader, dsc* desc) desc->dsc_length = sizeof(Decimal128); break; - case blr_dec_fixed: - desc->dsc_dtype = dtype_dec_fixed; - desc->dsc_length = sizeof(DecimalFixed); + case blr_int128: + desc->dsc_dtype = dtype_int128; + desc->dsc_length = sizeof(Int128); desc->dsc_scale = (int) blrReader.getByte(); break; diff --git a/src/jrd/sort.cpp b/src/jrd/sort.cpp index 947db4d47b..f893fb044d 100644 --- a/src/jrd/sort.cpp +++ b/src/jrd/sort.cpp @@ -804,6 +804,7 @@ void Sort::diddleKey(UCHAR* record, bool direction, bool duplicateHandling) case SKD_timestamp: case SKD_sql_date: case SKD_int64: + case SKD_int128: *p ^= 1 << 7; break; @@ -1047,6 +1048,24 @@ void Sort::diddleKey(UCHAR* record, bool direction, bool duplicateHandling) SWAP_LONGS(lwp[0], lwp[1], lw); break; + case SKD_int128: + // INT128 fits in four long, and hence two swaps should happen + // here for the right order comparison using DO_32_COMPARE + if (!direction) + { + SWAP_LONGS(lwp[0], lwp[3], lw); + SWAP_LONGS(lwp[1], lwp[2], lw); + } + + p[15] ^= 1 << 7; + + if (direction) + { + SWAP_LONGS(lwp[0], lwp[3], lw); + SWAP_LONGS(lwp[1], lwp[2], lw); + } + break; + #ifdef IEEE case SKD_double: if (!direction) diff --git a/src/jrd/sort.h b/src/jrd/sort.h index 47a72abd7e..eade49d674 100644 --- a/src/jrd/sort.h +++ b/src/jrd/sort.h @@ -145,6 +145,7 @@ const int SKD_dec64 = 16; const int SKD_dec128 = 17; const int SKD_sql_time_tz = 18; const int SKD_timestamp_tz = 19; +const int SKD_int128 = 20; // skd_flags const UCHAR SKD_ascending = 0; // default initializer diff --git a/src/jrd/val.h b/src/jrd/val.h index bbbb263278..0db02aacce 100644 --- a/src/jrd/val.h +++ b/src/jrd/val.h @@ -85,7 +85,7 @@ struct impure_value double vlu_double; Firebird::Decimal64 vlu_dec64; Firebird::Decimal128 vlu_dec128; - Firebird::DecimalFixed vlu_dec_fixed; + Firebird::Int128 vlu_int128; GDS_TIMESTAMP vlu_timestamp; ISC_TIMESTAMP_TZ vlu_timestamp_tz; GDS_TIME vlu_sql_time; @@ -101,7 +101,7 @@ struct impure_value void make_int64(const SINT64 val, const signed char scale = 0); void make_double(const double val); void make_decimal128(const Firebird::Decimal128 val); - void make_decimal_fixed(const Firebird::DecimalFixed val, const signed char scale); + void make_decimal_fixed(const Firebird::Int128 val, const signed char scale); }; // Do not use these methods where dsc_sub_type is not explicitly set to zero. @@ -145,14 +145,14 @@ inline void impure_value::make_decimal128(const Firebird::Decimal128 val) this->vlu_desc.dsc_address = reinterpret_cast(&this->vlu_misc.vlu_dec128); } -inline void impure_value::make_decimal_fixed(const Firebird::DecimalFixed val, const signed char scale) +inline void impure_value::make_decimal_fixed(const Firebird::Int128 val, const signed char scale) { - this->vlu_misc.vlu_dec_fixed = val; - this->vlu_desc.dsc_dtype = dtype_dec_fixed; - this->vlu_desc.dsc_length = sizeof(Firebird::DecimalFixed); + this->vlu_misc.vlu_int128 = val; + this->vlu_desc.dsc_dtype = dtype_int128; + this->vlu_desc.dsc_length = sizeof(Firebird::Int128); this->vlu_desc.dsc_scale = scale; this->vlu_desc.dsc_sub_type = 0; - this->vlu_desc.dsc_address = reinterpret_cast(&this->vlu_misc.vlu_dec_fixed); + this->vlu_desc.dsc_address = reinterpret_cast(&this->vlu_misc.vlu_int128); } struct impure_value_ex : public impure_value diff --git a/src/misc/pascal/Pascal.interface.pas b/src/misc/pascal/Pascal.interface.pas index ec5d396964..cf9ef252e0 100644 --- a/src/misc/pascal/Pascal.interface.pas +++ b/src/misc/pascal/Pascal.interface.pas @@ -17,7 +17,7 @@ ISC_QUAD = array [1..2] of Integer; FB_DEC16 = array [1..1] of Int64; FB_DEC34 = array [1..2] of Int64; - FB_DEC_FIXED = array [1..2] of Int64; + FB_INT128 = array [1..2] of Int64; isc_tr_handle = ^integer32; isc_stmt_handle = ^integer32; diff --git a/src/remote/client/BlrFromMessage.cpp b/src/remote/client/BlrFromMessage.cpp index e486d37377..4f1f127fed 100644 --- a/src/remote/client/BlrFromMessage.cpp +++ b/src/remote/client/BlrFromMessage.cpp @@ -127,10 +127,10 @@ void BlrFromMessage::buildBlr(IMessageMetadata* metadata) dtype = dtype_dec128; break; - case SQL_DEC_FIXED: - appendUChar(blr_dec_fixed); + case SQL_INT128: + appendUChar(blr_int128); appendUChar(scale); - dtype = dtype_dec_fixed; + dtype = dtype_int128; break; case SQL_DOUBLE: diff --git a/src/remote/parser.cpp b/src/remote/parser.cpp index de7c3a76af..13cb500afa 100644 --- a/src/remote/parser.cpp +++ b/src/remote/parser.cpp @@ -300,11 +300,11 @@ static rem_fmt* parse_format(const UCHAR*& blr, size_t& blr_length) align = type_alignments[dtype_dec128]; break; - case blr_dec_fixed: - desc->dsc_dtype = dtype_dec_fixed; - desc->dsc_length = sizeof(DecimalFixed); + case blr_int128: + desc->dsc_dtype = dtype_int128; + desc->dsc_length = sizeof(Int128); desc->dsc_scale = *blr++; - align = type_alignments[dtype_dec_fixed]; + align = type_alignments[dtype_int128]; break; // this case cannot occur as switch paramater is char and blr_blob diff --git a/src/utilities/ntrace/TracePluginImpl.cpp b/src/utilities/ntrace/TracePluginImpl.cpp index 5f411df60f..f9d0bd7e97 100644 --- a/src/utilities/ntrace/TracePluginImpl.cpp +++ b/src/utilities/ntrace/TracePluginImpl.cpp @@ -762,7 +762,7 @@ void TracePluginImpl::appendParams(ITraceParams* params) case dtype_dec128: paramtype = "decfloat(34)"; break; - case dtype_dec_fixed: + case dtype_int128: paramtype = "decimal"; break; @@ -887,11 +887,10 @@ void TracePluginImpl::appendParams(ITraceParams* params) ((Decimal128*) parameters->dsc_address)->toString(paramvalue); break; - case dtype_dec_fixed: + case dtype_int128: try { - DecimalStatus decSt(FB_DEC_Errors); - ((DecimalFixed*) parameters->dsc_address)->toString(decSt, parameters->dsc_scale, paramvalue); + ((Int128*) parameters->dsc_address)->toString(parameters->dsc_scale, paramvalue); } catch (const Exception& ex) { diff --git a/src/yvalve/YObjects.h b/src/yvalve/YObjects.h index ec52aa27f9..013bbe9adc 100644 --- a/src/yvalve/YObjects.h +++ b/src/yvalve/YObjects.h @@ -686,6 +686,7 @@ public: void encodeTimeStampTz(Firebird::CheckStatusWrapper* status, ISC_TIMESTAMP_TZ* timeStampTz, unsigned year, unsigned month, unsigned day, unsigned hours, unsigned minutes, unsigned seconds, unsigned fractions, const char* timeZone); + Firebird::IInt128* getInt128(Firebird::CheckStatusWrapper* status); }; } // namespace Why diff --git a/src/yvalve/gds.cpp b/src/yvalve/gds.cpp index 44282a7438..c0dd855594 100644 --- a/src/yvalve/gds.cpp +++ b/src/yvalve/gds.cpp @@ -60,6 +60,7 @@ #include "../common/classes/TempFile.h" #include "../common/utils_proto.h" #include "../common/ThreadStart.h" +#include "../common/Int128.h" #ifdef HAVE_UNISTD_H #include @@ -3034,9 +3035,9 @@ static int blr_print_dtype(gds_ctl* control) length = sizeof(Firebird::Decimal64); break; - case blr_dec_fixed: - string = "dec_fixed"; - length = sizeof(Firebird::DecimalFixed); + case blr_int128: + string = "int128"; + length = sizeof(Firebird::Int128); break; case blr_domain_name: @@ -3100,7 +3101,7 @@ static int blr_print_dtype(gds_ctl* control) case blr_long: case blr_quad: case blr_int64: - case blr_dec_fixed: + case blr_int128: blr_print_byte(control); break; diff --git a/src/yvalve/utl.cpp b/src/yvalve/utl.cpp index 7a4ab4fd76..8d0f32c34a 100644 --- a/src/yvalve/utl.cpp +++ b/src/yvalve/utl.cpp @@ -1303,6 +1303,49 @@ IDecFloat34* UtilInterface::getDecFloat34(CheckStatusWrapper* status) return &decFloat34; } +class IfaceInt128 FB_FINAL : public AutoIface > +{ +public: + // IInt128 implementation + void toString(CheckStatusWrapper* status, const FB_I128* from, int scale, unsigned bufSize, char* buffer) + { + try + { + const Int128* i128 = (Int128*)from; + i128->toString(scale, bufSize, buffer); + } + catch (const Exception& ex) + { + ex.stuffException(status); + } + } + + void fromString(CheckStatusWrapper* status, int scale, const char* from, FB_I128* to) + { + try + { + Int128* i128 = (Int128*)to; + scale -= CVT_decompose(from, static_cast(strlen(from)), i128, errorFunction); + i128->setScale(scale); + } + catch (const Exception& ex) + { + ex.stuffException(status); + } + } + + static void errorFunction(const Arg::StatusVector& v) + { + v.raise(); + } +}; + +IInt128* UtilInterface::getInt128(CheckStatusWrapper* status) +{ + static IfaceInt128 ifaceInt128; + return &ifaceInt128; +} + unsigned UtilInterface::setOffsets(CheckStatusWrapper* status, IMessageMetadata* metadata, IOffsetsCallback* callback) { From 4ac6e0cb45f0b10b5751f775cd8233d99bbeda73 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Tue, 17 Sep 2019 00:03:58 +0000 Subject: [PATCH 037/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 7cb6de1964..4e886ff85c 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1603 + FORMAL BUILD NUMBER:1604 */ -#define PRODUCT_VER_STRING "4.0.0.1603" -#define FILE_VER_STRING "WI-T4.0.0.1603" -#define LICENSE_VER_STRING "WI-T4.0.0.1603" -#define FILE_VER_NUMBER 4, 0, 0, 1603 +#define PRODUCT_VER_STRING "4.0.0.1604" +#define FILE_VER_STRING "WI-T4.0.0.1604" +#define LICENSE_VER_STRING "WI-T4.0.0.1604" +#define FILE_VER_NUMBER 4, 0, 0, 1604 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1603" +#define FB_BUILD_NO "1604" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 3a1dc0c9da..4105cc8e72 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1603 +BuildNum=1604 NowAt=`pwd` cd `dirname $0` From 26b287249b1aea1c1619e2024a1b0be564415aed Mon Sep 17 00:00:00 2001 From: hvlad Date: Tue, 17 Sep 2019 17:16:58 +0300 Subject: [PATCH 038/274] Fixed typo --- src/common/Int128.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/Int128.cpp b/src/common/Int128.cpp index 001e00534d..0735a5bca8 100644 --- a/src/common/Int128.cpp +++ b/src/common/Int128.cpp @@ -417,7 +417,7 @@ Int128 Int128::operator&=(ULONG mask) Int128 Int128::operator/(unsigned value) const { - Int128 rc; + Int128 rc(*this); rc.v.DivInt(value); return rc; } From 0fa41abc5dc3a31d676ced98a088f4cf615d0e47 Mon Sep 17 00:00:00 2001 From: Alex Peshkoff Date: Tue, 17 Sep 2019 18:34:28 +0300 Subject: [PATCH 039/274] Added int128 to rdb$types --- src/jrd/types.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jrd/types.h b/src/jrd/types.h index cfe38b0235..c117ebfbe5 100644 --- a/src/jrd/types.h +++ b/src/jrd/types.h @@ -45,6 +45,7 @@ TYPE("DECFLOAT(16)", blr_dec64, nam_f_type) TYPE("DECFLOAT(34)", blr_dec128, nam_f_type) TYPE("TIMESTAMP WITH TIMEZONE", blr_timestamp_tz, nam_f_type) TYPE("TIME WITH TIMEZONE", blr_sql_time_tz, nam_f_type) +TYPE("INT128", blr_int128, nam_f_type) TYPE("BINARY", 0, nam_f_sub_type) TYPE("TEXT", 1, nam_f_sub_type) From f558f13cd6a686ee60a084c475ba357d9d26da96 Mon Sep 17 00:00:00 2001 From: Dmitry Yemanov Date: Tue, 17 Sep 2019 20:27:27 +0300 Subject: [PATCH 040/274] Misc (indentation) --- src/jrd/idx.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/jrd/idx.cpp b/src/jrd/idx.cpp index 682506919e..1ee040fc48 100644 --- a/src/jrd/idx.cpp +++ b/src/jrd/idx.cpp @@ -1173,7 +1173,7 @@ static idx_e check_duplicates(thread_db* tdbb, // record index are the same, but for foreign keys they are different if (cmpRecordKeys(tdbb, rpb.rpb_record, relation_1, insertion_idx, - record, relation_2, record_idx)) + record, relation_2, record_idx)) { // When check foreign keys in snapshot or read consistency transaction, // ensure that master record is visible in transaction context and still @@ -1191,15 +1191,17 @@ static idx_e check_duplicates(thread_db* tdbb, continue; if (!cmpRecordKeys(tdbb, rpb.rpb_record, relation_1, insertion_idx, - record, relation_2, record_idx)) + record, relation_2, record_idx)) + { continue; + } + } } - } - result = idx_e_duplicate; - break; - } + result = idx_e_duplicate; + break; } + } } while (accessor.getNext()); delete rpb.rpb_record; From c2f4208699803e3edbb42666506b1c1133825c6c Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 18 Sep 2019 00:03:47 +0000 Subject: [PATCH 041/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 4e886ff85c..e9ffcc4f71 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1604 + FORMAL BUILD NUMBER:1607 */ -#define PRODUCT_VER_STRING "4.0.0.1604" -#define FILE_VER_STRING "WI-T4.0.0.1604" -#define LICENSE_VER_STRING "WI-T4.0.0.1604" -#define FILE_VER_NUMBER 4, 0, 0, 1604 +#define PRODUCT_VER_STRING "4.0.0.1607" +#define FILE_VER_STRING "WI-T4.0.0.1607" +#define LICENSE_VER_STRING "WI-T4.0.0.1607" +#define FILE_VER_NUMBER 4, 0, 0, 1607 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1604" +#define FB_BUILD_NO "1607" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 4105cc8e72..9be58da6d7 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1604 +BuildNum=1607 NowAt=`pwd` cd `dirname $0` From 128ecdc77f355ccfdcccf7c6d2c2c0231c410e72 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Thu, 19 Sep 2019 00:03:38 +0000 Subject: [PATCH 042/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index e9ffcc4f71..246840ea1e 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1607 + FORMAL BUILD NUMBER:1609 */ -#define PRODUCT_VER_STRING "4.0.0.1607" -#define FILE_VER_STRING "WI-T4.0.0.1607" -#define LICENSE_VER_STRING "WI-T4.0.0.1607" -#define FILE_VER_NUMBER 4, 0, 0, 1607 +#define PRODUCT_VER_STRING "4.0.0.1609" +#define FILE_VER_STRING "WI-T4.0.0.1609" +#define LICENSE_VER_STRING "WI-T4.0.0.1609" +#define FILE_VER_NUMBER 4, 0, 0, 1609 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1607" +#define FB_BUILD_NO "1609" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 9be58da6d7..6a268f7939 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1607 +BuildNum=1609 NowAt=`pwd` cd `dirname $0` From 093ba005e2aa0eefd9e1d61654581ac9d41b5bd7 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Wed, 18 Sep 2019 21:49:03 -0300 Subject: [PATCH 043/274] Fix CORE-6145 - Wrong result in "similar to" with non latin characters. --- src/jrd/Collation.cpp | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/src/jrd/Collation.cpp b/src/jrd/Collation.cpp index 64d05002d9..7bc350de70 100644 --- a/src/jrd/Collation.cpp +++ b/src/jrd/Collation.cpp @@ -116,6 +116,7 @@ public: Re2SimilarMatcher(thread_db* tdbb, MemoryPool& pool, TextType* textType, const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen) : PatternMatcher(pool, textType), + converter(INTL_convert_lookup(tdbb, CS_UTF8, textType->getCharSet()->getId())), buffer(pool) { UCharBuffer patternBuffer, escapeBuffer; @@ -131,8 +132,6 @@ public: flags |= (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ? SimilarToRegex::FLAG_CASE_INSENSITIVE : 0; - CsConvert converter = INTL_convert_lookup(tdbb, CS_UTF8, charSetId); - converter.convert(patternLen, patternStr, patternBuffer); if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) @@ -190,13 +189,24 @@ public: virtual bool result() { - if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) - UnicodeUtil::utf8Normalize(buffer); + UCharBuffer utfBuffer; + const auto charSetId = textType->getCharSet()->getId(); + UCharBuffer* bufferPtr = &buffer; - return regex->matches((const char*) buffer.begin(), buffer.getCount()); + if (charSetId != CS_NONE && charSetId != CS_BINARY && charSetId != CS_UTF8) + { + converter.convert(buffer.getCount(), buffer.begin(), utfBuffer); + bufferPtr = &utfBuffer; + } + + if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + UnicodeUtil::utf8Normalize(*bufferPtr); + + return regex->matches((const char*) bufferPtr->begin(), bufferPtr->getCount()); } private: + CsConvert converter; AutoPtr regex; UCharBuffer buffer; }; @@ -207,6 +217,7 @@ public: Re2SubstringSimilarMatcher(thread_db* tdbb, MemoryPool& pool, TextType* textType, const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen) : BaseSubstringSimilarMatcher(pool, textType), + converter(INTL_convert_lookup(tdbb, CS_UTF8, textType->getCharSet()->getId())), buffer(pool), resultStart(0), resultLength(0) @@ -224,8 +235,6 @@ public: flags |= (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ? SubstringSimilarRegex::FLAG_CASE_INSENSITIVE : 0; - CsConvert converter = INTL_convert_lookup(tdbb, textType->getCharSet()->getId(), CS_UTF8); - converter.convert(patternLen, patternStr, patternBuffer); if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) @@ -289,10 +298,20 @@ public: virtual bool result() { - if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) - UnicodeUtil::utf8Normalize(buffer); + UCharBuffer utfBuffer; + const auto charSetId = textType->getCharSet()->getId(); + UCharBuffer* bufferPtr = &buffer; - return regex->matches((const char*) buffer.begin(), buffer.getCount(), &resultStart, &resultLength); + if (charSetId != CS_NONE && charSetId != CS_BINARY && charSetId != CS_UTF8) + { + converter.convert(buffer.getCount(), buffer.begin(), utfBuffer); + bufferPtr = &utfBuffer; + } + + if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + UnicodeUtil::utf8Normalize(*bufferPtr); + + return regex->matches((const char*) bufferPtr->begin(), bufferPtr->getCount(), &resultStart, &resultLength); } virtual void getResultInfo(unsigned* start, unsigned* length) @@ -302,6 +321,7 @@ public: } private: + CsConvert converter; AutoPtr regex; UCharBuffer buffer; unsigned resultStart, resultLength; From 0b6311769e64bf5e337014a5ec276cabe1f6efc4 Mon Sep 17 00:00:00 2001 From: Ilya Eremin Date: Thu, 19 Sep 2019 09:34:56 +0300 Subject: [PATCH 044/274] Fixed CORE-6144: Inconsistent behaviour of the NEW context variable in AFTER UPDATE OR DELETE triggers (#225) * Fixed CORE-6144: Inconsistent behaviour of the NEW context variable in AFTER UPDATE OR DELETE triggers * MET_current() is used to get relation current format --- src/jrd/exe.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jrd/exe.cpp b/src/jrd/exe.cpp index 8f899e24b5..f858437fc9 100644 --- a/src/jrd/exe.cpp +++ b/src/jrd/exe.cpp @@ -1093,11 +1093,11 @@ void EXE_execute_triggers(thread_db* tdbb, if (!is_db_trigger && (!old_rec || !new_rec)) { - const Record* const record = old_rec ? old_rec : new_rec; - fb_assert(record && record->getFormat()); + record_param* rpb = old_rpb ? old_rpb : new_rpb; + fb_assert(rpb && rpb->rpb_relation); // copy the record MemoryPool& pool = *tdbb->getDefaultPool(); - null_rec = FB_NEW_POOL(pool) Record(pool, record->getFormat()); + null_rec = FB_NEW_POOL(pool) Record(pool, MET_current(tdbb, rpb->rpb_relation)); // initialize all fields to missing null_rec->nullify(); } From 6add872a1ae46315c6f6e305191a940a71fda041 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Thu, 19 Sep 2019 13:04:52 -0300 Subject: [PATCH 045/274] Complement fix for CORE-6145 - Wrong result in "similar to" with non latin characters. --- src/burp/burp.cpp | 2 +- src/common/SimilarToRegex.cpp | 12 ++++++------ src/common/SimilarToRegex.h | 16 +++++++--------- src/jrd/Collation.cpp | 12 ++++++------ src/jrd/replication/Manager.cpp | 4 ++-- src/utilities/ntrace/TracePluginImpl.cpp | 4 ++-- 6 files changed, 24 insertions(+), 26 deletions(-) diff --git a/src/burp/burp.cpp b/src/burp/burp.cpp index dc72fb6c0d..d2b83167c0 100644 --- a/src/burp/burp.cpp +++ b/src/burp/burp.cpp @@ -2544,7 +2544,7 @@ void BurpGlobals::setupSkipData(const Firebird::string& regexp) BurpGlobals* tdgbl = BurpGlobals::getSpecific(); skipDataMatcher.reset(FB_NEW_POOL(tdgbl->getPool()) Firebird::SimilarToRegex( - tdgbl->getPool(), Firebird::SimilarToRegex::FLAG_CASE_INSENSITIVE, + tdgbl->getPool(), Firebird::SimilarToFlag::CASE_INSENSITIVE, filter.c_str(), filter.length(), "\\", 1)); } diff --git a/src/common/SimilarToRegex.cpp b/src/common/SimilarToRegex.cpp index bb4e8f9565..ada659b2a6 100644 --- a/src/common/SimilarToRegex.cpp +++ b/src/common/SimilarToRegex.cpp @@ -772,9 +772,9 @@ SimilarToRegex::SimilarToRegex(MemoryPool& pool, unsigned flags, { SimilarToCompiler compiler(pool, regexp, COMP_FLAG_GROUP_CAPTURE | COMP_FLAG_PREFER_FEWER | - ((flags & FLAG_CASE_INSENSITIVE) ? COMP_FLAG_CASE_INSENSITIVE : 0) | - ((flags & FLAG_LATIN) ? COMP_FLAG_LATIN : 0) | - ((flags & FLAG_WELLFORMED) ? COMP_FLAG_WELLFORMED : 0), + ((flags & SimilarToFlag::CASE_INSENSITIVE) ? COMP_FLAG_CASE_INSENSITIVE : 0) | + ((flags & SimilarToFlag::LATIN) ? COMP_FLAG_LATIN : 0) | + ((flags & SimilarToFlag::WELLFORMED) ? COMP_FLAG_WELLFORMED : 0), patternStr, patternLen, escapeStr, escapeLen); finalizer = pool.registerFinalizer(finalize, this); @@ -844,9 +844,9 @@ SubstringSimilarRegex::SubstringSimilarRegex(MemoryPool& pool, unsigned flags, : PermanentStorage(pool) { SubstringSimilarCompiler compiler(pool, regexp, - ((flags & FLAG_CASE_INSENSITIVE) ? COMP_FLAG_CASE_INSENSITIVE : 0) | - ((flags & FLAG_LATIN) ? COMP_FLAG_LATIN : 0) | - ((flags & FLAG_WELLFORMED) ? COMP_FLAG_WELLFORMED : 0), + ((flags & SimilarToFlag::CASE_INSENSITIVE) ? COMP_FLAG_CASE_INSENSITIVE : 0) | + ((flags & SimilarToFlag::LATIN) ? COMP_FLAG_LATIN : 0) | + ((flags & SimilarToFlag::WELLFORMED) ? COMP_FLAG_WELLFORMED : 0), patternStr, patternLen, escapeStr, escapeLen); finalizer = pool.registerFinalizer(finalize, this); diff --git a/src/common/SimilarToRegex.h b/src/common/SimilarToRegex.h index 2834f82c99..89b8121a1a 100644 --- a/src/common/SimilarToRegex.h +++ b/src/common/SimilarToRegex.h @@ -30,13 +30,16 @@ namespace Firebird { +namespace SimilarToFlag +{ + static const unsigned CASE_INSENSITIVE = 0x1; + static const unsigned LATIN = 0x2; + static const unsigned WELLFORMED = 0x4; +}; + class SimilarToRegex : public PermanentStorage { public: - static const unsigned FLAG_CASE_INSENSITIVE = 0x1; - static const unsigned FLAG_LATIN = 0x2; - static const unsigned FLAG_WELLFORMED = 0x3; - struct MatchPos { unsigned start; @@ -65,11 +68,6 @@ private: // - Return S2. class SubstringSimilarRegex : public PermanentStorage { -public: - static const unsigned FLAG_CASE_INSENSITIVE = SimilarToRegex::FLAG_CASE_INSENSITIVE; - static const unsigned FLAG_LATIN = SimilarToRegex::FLAG_LATIN; - static const unsigned FLAG_WELLFORMED = SimilarToRegex::FLAG_WELLFORMED; - public: SubstringSimilarRegex(MemoryPool& pool, unsigned flags, const char* patternStr, unsigned patternLen, const char* escapeStr, unsigned escapeLen); diff --git a/src/jrd/Collation.cpp b/src/jrd/Collation.cpp index 7bc350de70..a3dd182c91 100644 --- a/src/jrd/Collation.cpp +++ b/src/jrd/Collation.cpp @@ -127,10 +127,10 @@ public: if (charSetId != CS_NONE && charSetId != CS_BINARY) { if (charSetId != CS_UTF8) - flags |= SimilarToRegex::FLAG_WELLFORMED; + flags |= SimilarToFlag::WELLFORMED; flags |= (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ? - SimilarToRegex::FLAG_CASE_INSENSITIVE : 0; + SimilarToFlag::CASE_INSENSITIVE : 0; converter.convert(patternLen, patternStr, patternBuffer); @@ -152,7 +152,7 @@ public: } } else - flags |= SimilarToRegex::FLAG_LATIN; + flags |= SimilarToFlag::LATIN; regex = FB_NEW_POOL(pool) SimilarToRegex(pool, flags, (const char*) patternStr, patternLen, @@ -230,10 +230,10 @@ public: if (charSetId != CS_NONE && charSetId != CS_BINARY) { if (charSetId != CS_UTF8) - flags |= SubstringSimilarRegex::FLAG_WELLFORMED; + flags |= SimilarToFlag::WELLFORMED; flags |= (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ? - SubstringSimilarRegex::FLAG_CASE_INSENSITIVE : 0; + SimilarToFlag::CASE_INSENSITIVE : 0; converter.convert(patternLen, patternStr, patternBuffer); @@ -255,7 +255,7 @@ public: } } else - flags |= SubstringSimilarRegex::FLAG_LATIN; + flags |= SimilarToFlag::LATIN; regex = FB_NEW_POOL(pool) SubstringSimilarRegex(pool, flags, (const char*) patternStr, patternLen, diff --git a/src/jrd/replication/Manager.cpp b/src/jrd/replication/Manager.cpp index 6a232329dd..5f6c8ee7ac 100644 --- a/src/jrd/replication/Manager.cpp +++ b/src/jrd/replication/Manager.cpp @@ -53,7 +53,7 @@ TableMatcher::TableMatcher(MemoryPool& pool, if (includeFilter.hasData()) { m_includeMatcher.reset(FB_NEW_POOL(pool) SimilarToRegex( - pool, SimilarToRegex::FLAG_CASE_INSENSITIVE, + pool, SimilarToFlag::CASE_INSENSITIVE, includeFilter.c_str(), includeFilter.length(), "\\", 1)); } @@ -61,7 +61,7 @@ TableMatcher::TableMatcher(MemoryPool& pool, if (excludeFilter.hasData()) { m_excludeMatcher.reset(FB_NEW_POOL(pool) SimilarToRegex( - pool, SimilarToRegex::FLAG_CASE_INSENSITIVE, + pool, SimilarToFlag::CASE_INSENSITIVE, excludeFilter.c_str(), excludeFilter.length(), "\\", 1)); } diff --git a/src/utilities/ntrace/TracePluginImpl.cpp b/src/utilities/ntrace/TracePluginImpl.cpp index f9d0bd7e97..4fa3d66c0e 100644 --- a/src/utilities/ntrace/TracePluginImpl.cpp +++ b/src/utilities/ntrace/TracePluginImpl.cpp @@ -134,7 +134,7 @@ TracePluginImpl::TracePluginImpl(IPluginBase* plugin, ISC_systemToUtf8(filter); include_matcher = FB_NEW SimilarToRegex( - *getDefaultMemoryPool(), SimilarToRegex::FLAG_CASE_INSENSITIVE, + *getDefaultMemoryPool(), SimilarToFlag::CASE_INSENSITIVE, filter.c_str(), filter.length(), "\\", 1); } @@ -146,7 +146,7 @@ TracePluginImpl::TracePluginImpl(IPluginBase* plugin, ISC_systemToUtf8(filter); exclude_matcher = FB_NEW SimilarToRegex( - *getDefaultMemoryPool(), SimilarToRegex::FLAG_CASE_INSENSITIVE, + *getDefaultMemoryPool(), SimilarToFlag::CASE_INSENSITIVE, filter.c_str(), filter.length(), "\\", 1); } From 51ee150915ca8ab2c287ff214532cdff05f62328 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Thu, 19 Sep 2019 13:35:33 -0300 Subject: [PATCH 046/274] CORE-6145 - Fix Windows build. --- src/utilities/ntrace/TraceConfiguration.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utilities/ntrace/TraceConfiguration.cpp b/src/utilities/ntrace/TraceConfiguration.cpp index e28d5a5c46..7daabd3126 100644 --- a/src/utilities/ntrace/TraceConfiguration.cpp +++ b/src/utilities/ntrace/TraceConfiguration.cpp @@ -134,7 +134,7 @@ void TraceCfgReader::readConfig() try { #ifdef WIN_NT // !CASE_SENSITIVITY - const unsigned regexFlags = SimilarToRegex::FLAG_CASE_INSENSITIVE; + const unsigned regexFlags = SimilarToFlag::CASE_INSENSITIVE; #else const unsigned regexFlags = 0; #endif From 99e1dbff4ffb077f10d1865f4e5168be6dfe21f2 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Fri, 20 Sep 2019 00:03:44 +0000 Subject: [PATCH 047/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 246840ea1e..a366513fa6 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1609 + FORMAL BUILD NUMBER:1613 */ -#define PRODUCT_VER_STRING "4.0.0.1609" -#define FILE_VER_STRING "WI-T4.0.0.1609" -#define LICENSE_VER_STRING "WI-T4.0.0.1609" -#define FILE_VER_NUMBER 4, 0, 0, 1609 +#define PRODUCT_VER_STRING "4.0.0.1613" +#define FILE_VER_STRING "WI-T4.0.0.1613" +#define LICENSE_VER_STRING "WI-T4.0.0.1613" +#define FILE_VER_NUMBER 4, 0, 0, 1613 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1609" +#define FB_BUILD_NO "1613" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 6a268f7939..4f0684b8e7 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1609 +BuildNum=1613 NowAt=`pwd` cd `dirname $0` From d5ff48ecf20e671e1d767aa5c4d1f48f85c34885 Mon Sep 17 00:00:00 2001 From: Alex Peshkoff Date: Fri, 20 Sep 2019 13:30:36 +0300 Subject: [PATCH 048/274] Fixed use of 128-bit integers as literals --- src/common/Int128.cpp | 5 ++++ src/common/Int128.h | 1 + src/dsql/ExprNodes.cpp | 9 +++++-- src/dsql/Parser.cpp | 57 ++++++++++++++++++++++++++++++++---------- src/dsql/make.cpp | 12 +++++---- src/dsql/make_proto.h | 3 ++- src/dsql/parse.y | 10 +++++--- 7 files changed, 72 insertions(+), 25 deletions(-) diff --git a/src/common/Int128.cpp b/src/common/Int128.cpp index 0735a5bca8..87bbd1618f 100644 --- a/src/common/Int128.cpp +++ b/src/common/Int128.cpp @@ -444,6 +444,11 @@ bool Int128::operator>(Int128 value) const return v > value.v; } +bool Int128::operator>=(Int128 value) const +{ + return v >= value.v; +} + bool Int128::operator==(Int128 value) const { return v == value.v; diff --git a/src/common/Int128.h b/src/common/Int128.h index 758e387174..3fa5217663 100644 --- a/src/common/Int128.h +++ b/src/common/Int128.h @@ -89,6 +89,7 @@ public: int compare(Int128 tgt) const; bool operator>(Int128 value) const; + bool operator>=(Int128 value) const; bool operator==(Int128 value) const; int sign() const; diff --git a/src/dsql/ExprNodes.cpp b/src/dsql/ExprNodes.cpp index b7d2263625..9648041274 100644 --- a/src/dsql/ExprNodes.cpp +++ b/src/dsql/ExprNodes.cpp @@ -7373,6 +7373,7 @@ DmlNode* LiteralNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* case dtype_double: case dtype_dec128: + case dtype_int128: { // The double literal could potentially be used for any numeric literal - the value is // passed as if it were a text string. Convert the numeric string to its binary value @@ -7394,6 +7395,9 @@ DmlNode* LiteralNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* case dtype_dec128: node->litDesc.dsc_length = sizeof(Decimal128); break; + case dtype_int128: + node->litDesc.dsc_length = sizeof(Int128); + break; case dtype_long: node->litDesc.dsc_length = sizeof(SLONG); break; @@ -7516,6 +7520,7 @@ void LiteralNode::genConstant(DsqlCompilerScratch* dsqlScratch, const dsc* desc, case dtype_double: case dtype_dec128: + case dtype_int128: { // this is used for approximate/large numeric literal // which is transmitted to the engine as a string. @@ -7885,7 +7890,7 @@ dsc* LiteralNode::execute(thread_db* /*tdbb*/, jrd_req* /*request*/) const void LiteralNode::fixMinSInt64(MemoryPool& pool) { - // MIN_SINT64 should be stored as BIGINT, not DECFLOAT + // MIN_SINT64 should be stored as BIGINT, not 128-bit integer const UCHAR* s = litDesc.dsc_address; const char* minSInt64 = "9223372036854775808"; @@ -8561,7 +8566,7 @@ NegateNode::NegateNode(MemoryPool& pool, ValueExprNode* aArg) arg(aArg) { LiteralNode* literal = nodeAs(arg); - if (literal && literal->litDesc.dsc_dtype == dtype_dec128) + if (literal && literal->litDesc.dsc_dtype == dtype_int128) literal->fixMinSInt64(pool); } diff --git a/src/dsql/Parser.cpp b/src/dsql/Parser.cpp index 7d2bacdbb0..101bcefaa9 100644 --- a/src/dsql/Parser.cpp +++ b/src/dsql/Parser.cpp @@ -958,7 +958,9 @@ int Parser::yylexAux() bool have_exp_digit = false; // digit ... [eE] ... digit bool have_overflow = false; // value of digits > MAX_SINT64 bool positive_overflow = false; // number is exactly (MAX_SINT64 + 1) + bool have_128_over = false; // value of digits > MAX_INT128 FB_UINT64 number = 0; + Int128 num128; int expVal = 0; FB_UINT64 limit_by_10 = MAX_SINT64 / 10; int scale = 0; @@ -1029,16 +1031,31 @@ int Parser::yylexAux() if ((number > limit_by_10) || (c >= '8')) { have_overflow = true; + fb_assert(number <= MAX_SINT64); + num128.set((SINT64)number, 0); if ((number == limit_by_10) && (c == '8')) positive_overflow = true; } } } else + { positive_overflow = false; + if (!have_128_over) + { + static const CInt128 MAX_BY10(MAX_Int128 / 10); + if ((num128 >= MAX_BY10) && ((num128 > MAX_BY10) || (c >= '8'))) + have_128_over = true; + } + } if (!have_overflow) number = number * 10 + (c - '0'); + else if (!have_128_over) + { + num128 *= 10; + num128 += (c - '0'); + } if (have_decimal) --scale; @@ -1064,6 +1081,7 @@ int Parser::yylexAux() { have_overflow = true; positive_overflow = false; + have_128_over = true; } // check for a more complex overflow case @@ -1075,35 +1093,48 @@ int Parser::yylexAux() { have_overflow = true; positive_overflow = false; + have_128_over = true; } } - // Should we use floating point type? - if (have_exp_digit || have_overflow || positive_overflow) + // Special case - on the boarder of positive number + if (positive_overflow) { - if (positive_overflow && scale) - { - yylval.lim64ptr = newLim64String( - Firebird::string(lex.last_token, lex.ptr - lex.last_token), scale); - lex.last_token_bk = lex.last_token; - lex.line_start_bk = lex.line_start; - lex.lines_bk = lex.lines; + yylval.lim64ptr = newLim64String( + Firebird::string(lex.last_token, lex.ptr - lex.last_token), scale); + lex.last_token_bk = lex.last_token; + lex.line_start_bk = lex.line_start; + lex.lines_bk = lex.lines; - return TOK_LIMIT64_NUMBER; - } + return scale ? TOK_LIMIT64_NUMBER : TOK_LIMIT64_INT; + } + // Should we use floating point type? + if (have_exp_digit || have_128_over) + { yylval.stringPtr = newString( Firebird::string(lex.last_token, lex.ptr - lex.last_token)); lex.last_token_bk = lex.last_token; lex.line_start_bk = lex.line_start; lex.lines_bk = lex.lines; - return positive_overflow ? TOK_LIMIT64_INT : have_overflow ? TOK_DECIMAL_NUMBER : TOK_FLOAT_NUMBER; + return have_overflow ? TOK_DECIMAL_NUMBER : TOK_FLOAT_NUMBER; + } + + // May be 128-bit integer? + if (have_overflow) + { + yylval.lim64ptr = newLim64String( + Firebird::string(lex.last_token, lex.ptr - lex.last_token), scale); + lex.last_token_bk = lex.last_token; + lex.line_start_bk = lex.line_start; + lex.lines_bk = lex.lines; + + return TOK_NUM128; } if (!have_exp) { - // We should return some kind (scaled-) integer type // except perhaps in dialect 1. diff --git a/src/dsql/make.cpp b/src/dsql/make.cpp index 3be3b340a9..877c6ff741 100644 --- a/src/dsql/make.cpp +++ b/src/dsql/make.cpp @@ -180,16 +180,16 @@ LiteralNode* MAKE_const_sint64(SINT64 value, SCHAR scale) @param numeric_flag **/ -ValueExprNode* MAKE_constant(const char* str, dsql_constant_type numeric_flag) +ValueExprNode* MAKE_constant(const char* str, dsql_constant_type numeric_flag, SSHORT scale) { thread_db* tdbb = JRD_get_thread_data(); - LiteralNode* literal = FB_NEW_POOL(*tdbb->getDefaultPool()) LiteralNode(*tdbb->getDefaultPool()); switch (numeric_flag) { case CONSTANT_DOUBLE: case CONSTANT_DECIMAL: + case CONSTANT_NUM128: // This is a numeric value which is transported to the engine as // a string. The engine will convert it. Use dtype_double/dec128 // so that the engine can distinguish it from an actual string. @@ -197,15 +197,17 @@ ValueExprNode* MAKE_constant(const char* str, dsql_constant_type numeric_flag) // to constants less than 32K - 1 bytes. Not real problem. { - literal->litDesc.dsc_dtype = numeric_flag == CONSTANT_DOUBLE ? dtype_double : dtype_dec128; - literal->litDesc.dsc_scale = 0; + literal->litDesc.dsc_dtype = numeric_flag == CONSTANT_DOUBLE ? dtype_double : + numeric_flag == CONSTANT_DECIMAL ? dtype_dec128 : dtype_int128; + literal->litDesc.dsc_scale = scale; size_t l = strlen(str); if (l > MAX_SSHORT) { ERRD_post(Arg::Gds(isc_imp_exc) << Arg::Gds(isc_num_literal)); } literal->litDesc.dsc_sub_type = static_cast(l); // Keep length in sub_type which is unused - literal->litDesc.dsc_length = numeric_flag == CONSTANT_DOUBLE ? sizeof(double) : sizeof(Decimal128); + literal->litDesc.dsc_length = numeric_flag == CONSTANT_DOUBLE ? sizeof(double) : + numeric_flag == CONSTANT_DECIMAL ? sizeof(Decimal128) : sizeof(Int128); literal->litDesc.dsc_address = (UCHAR*) str; } break; diff --git a/src/dsql/make_proto.h b/src/dsql/make_proto.h index d187ab2dc6..e506413de4 100644 --- a/src/dsql/make_proto.h +++ b/src/dsql/make_proto.h @@ -47,6 +47,7 @@ namespace Jrd { enum dsql_constant_type { CONSTANT_DOUBLE = 1, // stored as a string CONSTANT_DECIMAL, // stored as a string + CONSTANT_NUM128, // stored as a string CONSTANT_DATE, // stored as a SLONG CONSTANT_TIME, // stored as a ULONG CONSTANT_TIMESTAMP, // stored as a QUAD @@ -78,7 +79,7 @@ namespace Jrd { Jrd::LiteralNode* MAKE_const_slong(SLONG); Jrd::LiteralNode* MAKE_const_sint64(SINT64 value, SCHAR scale); -Jrd::ValueExprNode* MAKE_constant(const char*, Jrd::dsql_constant_type); +Jrd::ValueExprNode* MAKE_constant(const char*, Jrd::dsql_constant_type, SSHORT = 0); Jrd::LiteralNode* MAKE_str_constant(const Jrd::IntlString*, SSHORT); Jrd::FieldNode* MAKE_field(Jrd::dsql_ctx*, Jrd::dsql_fld*, Jrd::ValueListNode*); Jrd::FieldNode* MAKE_field_name(const char*); diff --git a/src/dsql/parse.y b/src/dsql/parse.y index 6a297f6606..7ceeed53ea 100644 --- a/src/dsql/parse.y +++ b/src/dsql/parse.y @@ -331,8 +331,8 @@ using namespace Firebird; %token WORK %token WRITE -%token FLOAT_NUMBER DECIMAL_NUMBER LIMIT64_INT -%token LIMIT64_NUMBER +%token FLOAT_NUMBER DECIMAL_NUMBER +%token LIMIT64_NUMBER LIMIT64_INT NUM128 %token SYMBOL %token NUMBER32BIT @@ -7418,9 +7418,11 @@ u_numeric_constant : ul_numeric_constant { $$ = $1; } | LIMIT64_NUMBER - { $$ = MAKE_constant($1->c_str(), CONSTANT_DECIMAL); } + { $$ = MAKE_constant($1->c_str(), CONSTANT_NUM128, $1->getScale()); } | LIMIT64_INT - { $$ = MAKE_constant($1->c_str(), CONSTANT_DECIMAL); } + { $$ = MAKE_constant($1->c_str(), CONSTANT_NUM128); } + | NUM128 + { $$ = MAKE_constant($1->c_str(), CONSTANT_NUM128, $1->getScale()); } ; %type ul_numeric_constant From 931ebc16dbfa2717d27db6873fc00da06aa14fac Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 21 Sep 2019 00:04:11 +0000 Subject: [PATCH 049/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index a366513fa6..619756ce76 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1613 + FORMAL BUILD NUMBER:1614 */ -#define PRODUCT_VER_STRING "4.0.0.1613" -#define FILE_VER_STRING "WI-T4.0.0.1613" -#define LICENSE_VER_STRING "WI-T4.0.0.1613" -#define FILE_VER_NUMBER 4, 0, 0, 1613 +#define PRODUCT_VER_STRING "4.0.0.1614" +#define FILE_VER_STRING "WI-T4.0.0.1614" +#define LICENSE_VER_STRING "WI-T4.0.0.1614" +#define FILE_VER_NUMBER 4, 0, 0, 1614 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1613" +#define FB_BUILD_NO "1614" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 4f0684b8e7..43fb6af94e 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1613 +BuildNum=1614 NowAt=`pwd` cd `dirname $0` From 2d41ad47c9934f44a174bde35f3f9fb389654f04 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Mon, 23 Sep 2019 13:18:42 +0300 Subject: [PATCH 050/274] Fixed 32-bit build --- src/common/Int128.cpp | 2 +- src/common/Int128.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/Int128.cpp b/src/common/Int128.cpp index 87bbd1618f..0b3f4b6b43 100644 --- a/src/common/Int128.cpp +++ b/src/common/Int128.cpp @@ -58,7 +58,7 @@ namespace Firebird { Int128 Int128::set(SLONG value, int scale) { - v = value; + v = ttmath::sint(value); setScale(scale); return *this; } diff --git a/src/common/Int128.h b/src/common/Int128.h index 3fa5217663..80c40dcb49 100644 --- a/src/common/Int128.h +++ b/src/common/Int128.h @@ -49,9 +49,9 @@ class Int128 //: public Decimal128Base { public: #if SIZEOF_LONG < 8 - Int128 set(int value) + Int128 set(int value, int scale) { - return set(SLONG(value)); + return set(SLONG(value), scale); } #endif Int128 set(SLONG value, int scale); From f30326d0c56ebc3bd3873fda2effda6701ed6573 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Tue, 24 Sep 2019 00:04:09 +0000 Subject: [PATCH 051/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 619756ce76..b5b341547c 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1614 + FORMAL BUILD NUMBER:1615 */ -#define PRODUCT_VER_STRING "4.0.0.1614" -#define FILE_VER_STRING "WI-T4.0.0.1614" -#define LICENSE_VER_STRING "WI-T4.0.0.1614" -#define FILE_VER_NUMBER 4, 0, 0, 1614 +#define PRODUCT_VER_STRING "4.0.0.1615" +#define FILE_VER_STRING "WI-T4.0.0.1615" +#define LICENSE_VER_STRING "WI-T4.0.0.1615" +#define FILE_VER_NUMBER 4, 0, 0, 1615 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1614" +#define FB_BUILD_NO "1615" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 43fb6af94e..3915c5fe15 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1614 +BuildNum=1615 NowAt=`pwd` cd `dirname $0` From 80bee2c52fa500d1b108d6241f19456429690110 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Tue, 24 Sep 2019 15:04:40 +0300 Subject: [PATCH 052/274] Fixed initialization of limits in cvt_decompose, also small optimization --- src/common/cvt.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/common/cvt.cpp b/src/common/cvt.cpp index debc9cff6f..462c7055c0 100644 --- a/src/common/cvt.cpp +++ b/src/common/cvt.cpp @@ -2771,9 +2771,9 @@ public: lb10 compareLimitBy10() { - if (value > Traits::UPPER_LIMIT / 10) + if (value > Traits::UPPER_LIMIT_BY_10) return RETVAL_OVERFLOW; - if (value == Traits::UPPER_LIMIT / 10) + if (value == Traits::UPPER_LIMIT_BY_10) return RETVAL_POSSIBLE_OVERFLOW; return RETVAL_NO_OVERFLOW; } @@ -2804,7 +2804,7 @@ class SLONGTraits { public: typedef SLONG ValueType; - static const SLONG UPPER_LIMIT = MAX_SLONG; + static const SLONG UPPER_LIMIT_BY_10 = MAX_SLONG / 10; static const SLONG LOWER_LIMIT = MIN_SLONG; }; @@ -2831,7 +2831,7 @@ class SINT64Traits { public: typedef SINT64 ValueType; - static const SINT64 UPPER_LIMIT = MAX_SINT64; + static const SINT64 UPPER_LIMIT_BY_10 = MAX_SINT64 / 10; static const SINT64 LOWER_LIMIT = MIN_SINT64; }; @@ -2858,12 +2858,12 @@ class I128Traits { public: typedef Int128 ValueType; - static const CInt128 UPPER_LIMIT; + static const CInt128 UPPER_LIMIT_BY_10; static const CInt128 LOWER_LIMIT; }; -const CInt128 I128Traits::UPPER_LIMIT(MAX_Int128); -const CInt128 I128Traits::LOWER_LIMIT(MIN_Int128); +const CInt128 I128Traits::UPPER_LIMIT_BY_10(CInt128(CInt128::MkMax) / 10); +const CInt128 I128Traits::LOWER_LIMIT(CInt128::MkMin); SSHORT CVT_decompose(const char* str, USHORT len, Int128* val, ErrorFunction err) { From ade10871dfdf2927b8b948208e37d56fc328d9c9 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 25 Sep 2019 00:04:24 +0000 Subject: [PATCH 053/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index b5b341547c..56e47cd13b 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1615 + FORMAL BUILD NUMBER:1616 */ -#define PRODUCT_VER_STRING "4.0.0.1615" -#define FILE_VER_STRING "WI-T4.0.0.1615" -#define LICENSE_VER_STRING "WI-T4.0.0.1615" -#define FILE_VER_NUMBER 4, 0, 0, 1615 +#define PRODUCT_VER_STRING "4.0.0.1616" +#define FILE_VER_STRING "WI-T4.0.0.1616" +#define LICENSE_VER_STRING "WI-T4.0.0.1616" +#define FILE_VER_NUMBER 4, 0, 0, 1616 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1615" +#define FB_BUILD_NO "1616" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 3915c5fe15..3706be44f2 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1615 +BuildNum=1616 NowAt=`pwd` cd `dirname $0` From 63dfdc50db89db97865cf2309a18d088f26e1ac8 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Thu, 26 Sep 2019 12:47:13 -0300 Subject: [PATCH 054/274] Postfix for 28e1874 (SIMILAR TO problems). Problem affects CORE-4599 fbt test. --- src/common/TextType.cpp | 4 ++-- src/common/TextType.h | 8 +++++++- src/jrd/Collation.cpp | 34 +++++++++++++++++----------------- src/jrd/Collation.h | 6 +++--- src/jrd/IntlManager.cpp | 1 - src/jrd/intl.cpp | 2 +- 6 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/common/TextType.cpp b/src/common/TextType.cpp index cd69aa5c80..ed16de3ba5 100644 --- a/src/common/TextType.cpp +++ b/src/common/TextType.cpp @@ -101,8 +101,8 @@ namespace Jrd { -TextType::TextType(TTYPE_ID _type, texttype *_tt, CharSet* _cs) - : tt(_tt), cs(_cs), type(_type) +TextType::TextType(TTYPE_ID _type, texttype *_tt, USHORT _attributes, CharSet* _cs) + : tt(_tt), cs(_cs), type(_type), attributes(_attributes) { if (cs->getSqlMatchAnyLength() != 0) { diff --git a/src/common/TextType.h b/src/common/TextType.h index bfe0e68a26..8befa294f6 100644 --- a/src/common/TextType.h +++ b/src/common/TextType.h @@ -41,7 +41,7 @@ class CharSet; class TextType { public: - TextType(TTYPE_ID _type, texttype *_tt, CharSet* _cs); + TextType(TTYPE_ID _type, texttype* _tt, USHORT _attributes, CharSet* _cs); private: TextType(const TextType&); // Not implemented @@ -82,6 +82,11 @@ public: return type; } + USHORT getAttributes() const + { + return attributes; + } + CharSet* getCharSet() const { return cs; @@ -99,6 +104,7 @@ protected: private: TTYPE_ID type; + USHORT attributes; public: enum diff --git a/src/jrd/Collation.cpp b/src/jrd/Collation.cpp index a3dd182c91..33f8a14f1e 100644 --- a/src/jrd/Collation.cpp +++ b/src/jrd/Collation.cpp @@ -129,12 +129,12 @@ public: if (charSetId != CS_UTF8) flags |= SimilarToFlag::WELLFORMED; - flags |= (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ? + flags |= (textType->getAttributes() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ? SimilarToFlag::CASE_INSENSITIVE : 0; converter.convert(patternLen, patternStr, patternBuffer); - if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) UnicodeUtil::utf8Normalize(patternBuffer); patternStr = patternBuffer.begin(); @@ -144,7 +144,7 @@ public: { converter.convert(escapeLen, escapeStr, escapeBuffer); - if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) UnicodeUtil::utf8Normalize(escapeBuffer); escapeStr = escapeBuffer.begin(); @@ -199,7 +199,7 @@ public: bufferPtr = &utfBuffer; } - if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) UnicodeUtil::utf8Normalize(*bufferPtr); return regex->matches((const char*) bufferPtr->begin(), bufferPtr->getCount()); @@ -232,12 +232,12 @@ public: if (charSetId != CS_UTF8) flags |= SimilarToFlag::WELLFORMED; - flags |= (textType->getFlags() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ? + flags |= (textType->getAttributes() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ? SimilarToFlag::CASE_INSENSITIVE : 0; converter.convert(patternLen, patternStr, patternBuffer); - if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) UnicodeUtil::utf8Normalize(patternBuffer); patternStr = patternBuffer.begin(); @@ -247,7 +247,7 @@ public: { converter.convert(escapeLen, escapeStr, escapeBuffer); - if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) UnicodeUtil::utf8Normalize(escapeBuffer); escapeStr = escapeBuffer.begin(); @@ -308,7 +308,7 @@ public: bufferPtr = &utfBuffer; } - if (textType->getFlags() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) + if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) UnicodeUtil::utf8Normalize(*bufferPtr); return regex->matches((const char*) bufferPtr->begin(), bufferPtr->getCount(), &resultStart, &resultLength); @@ -949,8 +949,8 @@ template < class CollationImpl : public Collation { public: - CollationImpl(TTYPE_ID a_type, texttype* a_tt, CharSet* a_cs) - : Collation(a_type, a_tt, a_cs) + CollationImpl(TTYPE_ID a_type, texttype* a_tt, USHORT a_attributes, CharSet* a_cs) + : Collation(a_type, a_tt, a_attributes, a_cs) { } @@ -1027,7 +1027,7 @@ public: }; template -Collation* newCollation(MemoryPool& pool, TTYPE_ID id, texttype* tt, CharSet* cs) +Collation* newCollation(MemoryPool& pool, TTYPE_ID id, texttype* tt, USHORT attributes, CharSet* cs) { using namespace Firebird; @@ -1052,9 +1052,9 @@ Collation* newCollation(MemoryPool& pool, TTYPE_ID id, texttype* tt, CharSet* cs > NonDirectImpl; if (tt->texttype_flags & TEXTTYPE_DIRECT_MATCH) - return FB_NEW_POOL(pool) DirectImpl(id, tt, cs); + return FB_NEW_POOL(pool) DirectImpl(id, tt, attributes, cs); else - return FB_NEW_POOL(pool) NonDirectImpl(id, tt, cs); + return FB_NEW_POOL(pool) NonDirectImpl(id, tt, attributes, cs); } } // namespace @@ -1066,18 +1066,18 @@ Collation* newCollation(MemoryPool& pool, TTYPE_ID id, texttype* tt, CharSet* cs namespace Jrd { -Collation* Collation::createInstance(MemoryPool& pool, TTYPE_ID id, texttype* tt, CharSet* cs) +Collation* Collation::createInstance(MemoryPool& pool, TTYPE_ID id, texttype* tt, USHORT attributes, CharSet* cs) { switch (tt->texttype_canonical_width) { case 1: - return newCollation(pool, id, tt, cs); + return newCollation(pool, id, tt, attributes, cs); case 2: - return newCollation(pool, id, tt, cs); + return newCollation(pool, id, tt, attributes, cs); case 4: - return newCollation(pool, id, tt, cs); + return newCollation(pool, id, tt, attributes, cs); } fb_assert(false); diff --git a/src/jrd/Collation.h b/src/jrd/Collation.h index 268411d5f2..1f19820f09 100644 --- a/src/jrd/Collation.h +++ b/src/jrd/Collation.h @@ -41,11 +41,11 @@ class BaseSubstringSimilarMatcher; class Collation : public TextType { public: - static Collation* createInstance(MemoryPool& pool, TTYPE_ID id, texttype* tt, CharSet* cs); + static Collation* createInstance(MemoryPool& pool, TTYPE_ID id, texttype* tt, USHORT attributes, CharSet* cs); protected: - Collation(TTYPE_ID id, texttype *a_tt, CharSet* a_cs) - : TextType(id, a_tt, a_cs), + Collation(TTYPE_ID id, texttype *a_tt, USHORT a_attributes, CharSet* a_cs) + : TextType(id, a_tt, a_attributes, a_cs), useCount(0), existenceLock(NULL), obsolete(false) diff --git a/src/jrd/IntlManager.cpp b/src/jrd/IntlManager.cpp index 4e4a517ac1..ccc9857e1d 100644 --- a/src/jrd/IntlManager.cpp +++ b/src/jrd/IntlManager.cpp @@ -654,7 +654,6 @@ bool IntlManager::lookupCollation(const string& collationName, attributes, specificAttributes, specificAttributesLen, ignoreAttributes, collationExternalInfo.configInfo.c_str())) { - tt->texttype_flags = attributes; return true; } } diff --git a/src/jrd/intl.cpp b/src/jrd/intl.cpp index a81c402b3d..6865b2b104 100644 --- a/src/jrd/intl.cpp +++ b/src/jrd/intl.cpp @@ -415,7 +415,7 @@ Collation* CharSetContainer::lookupCollation(thread_db* tdbb, USHORT tt_id) } } - charset_collations[id] = Collation::createInstance(*att->att_pool, tt_id, tt, charset); + charset_collations[id] = Collation::createInstance(*att->att_pool, tt_id, tt, info.attributes, charset); charset_collations[id]->name = info.collationName; // we don't need a lock in the charset From a2173b2abc4636e9fb7095509c67e39a85997eda Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Fri, 27 Sep 2019 00:03:39 +0000 Subject: [PATCH 055/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 56e47cd13b..6e57de9979 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1616 + FORMAL BUILD NUMBER:1617 */ -#define PRODUCT_VER_STRING "4.0.0.1616" -#define FILE_VER_STRING "WI-T4.0.0.1616" -#define LICENSE_VER_STRING "WI-T4.0.0.1616" -#define FILE_VER_NUMBER 4, 0, 0, 1616 +#define PRODUCT_VER_STRING "4.0.0.1617" +#define FILE_VER_STRING "WI-T4.0.0.1617" +#define LICENSE_VER_STRING "WI-T4.0.0.1617" +#define FILE_VER_NUMBER 4, 0, 0, 1617 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1616" +#define FB_BUILD_NO "1617" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 3706be44f2..8dd1702782 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1616 +BuildNum=1617 NowAt=`pwd` cd `dirname $0` From ecbb17bc169ed7a09d5b1d701e021bb6894cfb93 Mon Sep 17 00:00:00 2001 From: hvlad Date: Mon, 30 Sep 2019 14:36:12 +0300 Subject: [PATCH 056/274] Fixed bug CORE-6150 : Bugcheck when PK\UK\FK constraint check read record already marked as damaged --- src/jrd/vio.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/jrd/vio.cpp b/src/jrd/vio.cpp index aa628a91f5..78ae55e870 100644 --- a/src/jrd/vio.cpp +++ b/src/jrd/vio.cpp @@ -2547,6 +2547,12 @@ bool VIO_get_current(thread_db* tdbb, // Get data if there is data. + if (rpb->rpb_flags & rpb_damaged) + { + CCH_RELEASE(tdbb, &rpb->getWindow(tdbb)); + return false; + } + if (rpb->rpb_flags & rpb_deleted) CCH_RELEASE(tdbb, &rpb->getWindow(tdbb)); else From 36e055f6aa2c929da8ee4f76d8cffcb94d9c4e13 Mon Sep 17 00:00:00 2001 From: hvlad Date: Mon, 30 Sep 2019 14:36:36 +0300 Subject: [PATCH 057/274] This should fix broken LRU pending chain --- src/jrd/cch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jrd/cch.cpp b/src/jrd/cch.cpp index 12ffae9df8..58dff41e48 100644 --- a/src/jrd/cch.cpp +++ b/src/jrd/cch.cpp @@ -5159,8 +5159,8 @@ void requeueRecentlyUsed(BufferControl* bcb) QUE_DELETE(bdb->bdb_in_use); QUE_INSERT(bcb->bcb_in_use, bdb->bdb_in_use); - bdb->bdb_flags &= ~BDB_lru_chained; bdb->bdb_lru_chain = NULL; + bdb->bdb_flags &= ~BDB_lru_chained; } chain = bcb->bcb_lru_chain; From de1c92cc6f8d799b62b4c18d7870cd133c7cc264 Mon Sep 17 00:00:00 2001 From: hvlad Date: Mon, 30 Sep 2019 14:43:11 +0300 Subject: [PATCH 058/274] This should fix double close of srcBlob if extBlob->close() failed --- src/jrd/extds/ExtDS.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/jrd/extds/ExtDS.cpp b/src/jrd/extds/ExtDS.cpp index 6722001418..b3bba18b79 100644 --- a/src/jrd/extds/ExtDS.cpp +++ b/src/jrd/extds/ExtDS.cpp @@ -2426,14 +2426,15 @@ void Statement::putExtBlob(thread_db* tdbb, dsc& src, dsc& dst) } srcBlob->BLB_close(tdbb); + srcBlob = NULL; extBlob->close(tdbb); } catch (const Exception&) { - extBlob->cancel(tdbb); if (srcBlob) { srcBlob->BLB_close(tdbb); } + extBlob->cancel(tdbb); throw; } } From 781b51fa0d3ddda079a59b4014948468c1732ce2 Mon Sep 17 00:00:00 2001 From: hvlad Date: Mon, 30 Sep 2019 14:45:32 +0300 Subject: [PATCH 059/274] Fix typo --- builds/win32/make_boot.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builds/win32/make_boot.bat b/builds/win32/make_boot.bat index 882c0829e7..5683e08830 100644 --- a/builds/win32/make_boot.bat +++ b/builds/win32/make_boot.bat @@ -175,7 +175,7 @@ goto :EOF @ml64.exe /c /Fo %FB_TEMP_DIR%\..\%FB_OBJ_DIR%\common\ttmathuint_x86_64_msvc.obj %FB_ROOT_PATH%\extern\ttmath\ttmathuint_x86_64_msvc.asm if errorlevel 1 call :boot2 ttmath_%FB_OBJ_DIR% @call set_build_target.bat %* DEBUG -@echo Building decNumber (%FB_OBJ_DIR%)... +@echo Building ttmath (%FB_OBJ_DIR%)... @mkdir %FB_TEMP_DIR%\..\%FB_OBJ_DIR%\common 2>nul @ml64.exe /c /Zi /Fo %FB_TEMP_DIR%\..\%FB_OBJ_DIR%\common\ttmathuint_x86_64_msvc.obj %FB_ROOT_PATH%\extern\ttmath\ttmathuint_x86_64_msvc.asm if errorlevel 1 call :boot2 ttmath_%FB_OBJ_DIR% From 4e2ba14ff9d56f7d98b937cce436dfcdf2570cc3 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Tue, 1 Oct 2019 00:03:40 +0000 Subject: [PATCH 060/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 6e57de9979..1e111c47af 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1617 + FORMAL BUILD NUMBER:1621 */ -#define PRODUCT_VER_STRING "4.0.0.1617" -#define FILE_VER_STRING "WI-T4.0.0.1617" -#define LICENSE_VER_STRING "WI-T4.0.0.1617" -#define FILE_VER_NUMBER 4, 0, 0, 1617 +#define PRODUCT_VER_STRING "4.0.0.1621" +#define FILE_VER_STRING "WI-T4.0.0.1621" +#define LICENSE_VER_STRING "WI-T4.0.0.1621" +#define FILE_VER_NUMBER 4, 0, 0, 1621 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1617" +#define FB_BUILD_NO "1621" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 8dd1702782..cfc5c787ee 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1617 +BuildNum=1621 NowAt=`pwd` cd `dirname $0` From fab2e23593d5fd1b43d853fa639dbe09e6f19cd8 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sat, 5 Oct 2019 20:01:48 -0300 Subject: [PATCH 061/274] Postfix for 28e1874 (SIMILAR TO problems). Problem affects CORE-3523 fbt test and others. --- src/common/SimilarToRegex.cpp | 94 +++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/src/common/SimilarToRegex.cpp b/src/common/SimilarToRegex.cpp index ada659b2a6..364cf18f8c 100644 --- a/src/common/SimilarToRegex.cpp +++ b/src/common/SimilarToRegex.cpp @@ -356,16 +356,17 @@ namespace { const char* similarClass; const char* re2ClassInclude; - const char* re2ClassExclude; + const char* re2ClassExcludeUtf; + const char* re2ClassExcludeLatin; } static const classes[] = { - {"alnum", "[:alnum:]", "[:^alnum:]"}, - {"alpha", "[:alpha:]", "[:^alpha:]"}, - {"digit", "[:digit:]", "[:^digit:]"}, - {"lower", "[:lower:]", "[:^lower:]"}, - {"space", " ", "\\x00-\\x1F\\x21-\\x{10FFFF}"}, - {"upper", "[:upper:]", "[:^upper:]"}, - {"whitespace", "[:space:]", "[:^space:]"} + {"alnum", "[:alnum:]", "[:^alnum:]", "[:^alnum:]"}, + {"alpha", "[:alpha:]", "[:^alpha:]", "[:^alpha:]"}, + {"digit", "[:digit:]", "[:^digit:]", "[:^digit:]"}, + {"lower", "[:lower:]", "[:^lower:]", "[:^lower:]"}, + {"space", " ", "\\x00-\\x1F\\x21-\\x{10FFFF}", "\\x00-\\x1F\\x21-\\xFF"}, + {"upper", "[:upper:]", "[:^upper:]", "[:^upper:]"}, + {"whitespace", "[:space:]", "[:^space:]", "[:^space:]"} }; struct Item @@ -373,9 +374,12 @@ namespace int clazz; unsigned firstStart, firstEnd, lastStart, lastEnd; }; + + const UChar32 maxChar = (flags & COMP_FLAG_LATIN) ? 0xFF : 0x10FFFF; Array items; unsigned includeCount = 0; bool exclude = false; + bool invalidInclude = false; do { @@ -413,9 +417,7 @@ namespace } Item item; - - if (!exclude) - ++includeCount; + bool strip = false; if (charClass) { @@ -472,49 +474,68 @@ namespace item.lastStart = charSavePos; item.lastEnd = patternPos; + + int32_t cPos = item.firstStart; + UChar32 c1 = getChar(flags & COMP_FLAG_LATIN, patternStr, patternLen, cPos); + + cPos = item.lastStart; + UChar32 c2 = getChar(flags & COMP_FLAG_LATIN, patternStr, patternLen, cPos); + + strip = c1 > c2; } } - items.add(item); + if (strip) + { + if (!exclude) + invalidInclude = true; + } + else + { + if (!exclude) + ++includeCount; + + items.add(item); + } if (!hasPatternChar()) status_exception::raise(Arg::Gds(isc_invalid_similar_pattern)); } while (peekPatternChar() != ']'); + exclude = includeCount < items.getCount(); + auto appendItem = [&](const Item& item, bool negated) { if (item.clazz != -1) { re2PatternStr.append(negated ? - classes[item.clazz].re2ClassExclude : + (flags & COMP_FLAG_LATIN ? + classes[item.clazz].re2ClassExcludeLatin : + classes[item.clazz].re2ClassExcludeUtf + ) : classes[item.clazz].re2ClassInclude); } else { if (negated) { - char hex[20]; + char hex[40]; int32_t cPos = item.firstStart; UChar32 c = getChar(flags & COMP_FLAG_LATIN, patternStr, patternLen, cPos); if (c > 0) { - re2PatternStr.append("\\x00"); - re2PatternStr.append("-"); - - sprintf(hex, "\\x{%X}", (int) c - 1); + sprintf(hex, "\\x00-\\x{%X}", (int) c - 1); re2PatternStr.append(hex); } cPos = item.lastStart; c = getChar(flags & COMP_FLAG_LATIN, patternStr, patternLen, cPos); - if (c < 0x10FFFF) + if (c < maxChar) { - sprintf(hex, "\\x{%X}", (int) c + 1); + sprintf(hex, "\\x{%X}-\\x{%X}", (int) c + 1, maxChar); re2PatternStr.append(hex); - re2PatternStr.append("-"); - re2PatternStr.append("\\x{10FFFF}"); } } else @@ -560,15 +581,26 @@ namespace } else { - re2PatternStr.append("["); + if (items.hasData() && !(invalidInclude && includeCount == 0)) + { + re2PatternStr.append("["); - if (exclude) - re2PatternStr.append("^"); + if (exclude) + re2PatternStr.append("^"); - for (unsigned i = 0; i < items.getCount(); ++i) - appendItem(items[i], exclude && i < includeCount); + for (unsigned i = 0; i < items.getCount(); ++i) + appendItem(items[i], exclude && i < includeCount); - re2PatternStr.append("]"); + re2PatternStr.append("]"); + } + else if (invalidInclude) + { + char str[30]; + sprintf(str, "[^\\x{0}-\\x{%X}]", maxChar); + re2PatternStr.append(str); + } + else + re2PatternStr.append("."); } getPatternChar(); @@ -695,13 +727,13 @@ namespace AutoPtr regexp1, regexp2, regexp3; - SimilarToCompiler compiler1(pool, regexp1, COMP_FLAG_PREFER_FEWER, + SimilarToCompiler compiler1(pool, regexp1, COMP_FLAG_PREFER_FEWER | (flags & COMP_FLAG_LATIN), aPatternStr, positions[0] - escapeLen - 1, escapeStr, escapeLen); - SimilarToCompiler compiler2(pool, regexp2, 0, + SimilarToCompiler compiler2(pool, regexp2, (flags & COMP_FLAG_LATIN), aPatternStr + positions[0], positions[1] - positions[0] - escapeLen - 1, escapeStr, escapeLen); - SimilarToCompiler compiler3(pool, regexp3, COMP_FLAG_PREFER_FEWER, + SimilarToCompiler compiler3(pool, regexp3, COMP_FLAG_PREFER_FEWER | (flags & COMP_FLAG_LATIN), aPatternStr + positions[1], patternLen - positions[1], escapeStr, escapeLen); string finalRe2Pattern; From 6ef8731591e4c22440c3f7493eba4779195d2800 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sun, 6 Oct 2019 00:03:42 +0000 Subject: [PATCH 062/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 1e111c47af..8424ed4ac5 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1621 + FORMAL BUILD NUMBER:1622 */ -#define PRODUCT_VER_STRING "4.0.0.1621" -#define FILE_VER_STRING "WI-T4.0.0.1621" -#define LICENSE_VER_STRING "WI-T4.0.0.1621" -#define FILE_VER_NUMBER 4, 0, 0, 1621 +#define PRODUCT_VER_STRING "4.0.0.1622" +#define FILE_VER_STRING "WI-T4.0.0.1622" +#define LICENSE_VER_STRING "WI-T4.0.0.1622" +#define FILE_VER_NUMBER 4, 0, 0, 1622 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1621" +#define FB_BUILD_NO "1622" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index cfc5c787ee..96f2edb5b0 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1621 +BuildNum=1622 NowAt=`pwd` cd `dirname $0` From 6ab3b75d6e13510dc35f0d90ced05c58e03199ce Mon Sep 17 00:00:00 2001 From: hvlad Date: Mon, 7 Oct 2019 14:36:39 +0300 Subject: [PATCH 063/274] Fixed usage of RSP register. It should fix a crash reported by Mark in fb-devel (fbclient.dll on Windows 64 bit broken) --- extern/ttmath/ttmathuint_x86_64_msvc.asm | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extern/ttmath/ttmathuint_x86_64_msvc.asm b/extern/ttmath/ttmathuint_x86_64_msvc.asm index b7c85c2b84..aae113f636 100644 --- a/extern/ttmath/ttmathuint_x86_64_msvc.asm +++ b/extern/ttmath/ttmathuint_x86_64_msvc.asm @@ -151,12 +151,12 @@ ttmath_addindexed2_x64 PROC ; rdx = b (value size) ; r8 = nPos ; r9 = nValue1 - ; [esp+0x28] = nValue2 + ; [rsp+0x28] = nValue2 xor rax, rax ; return value mov r11, rcx ; table sub rdx, r8 ; rdx = remaining count of uints - mov r10, [esp+028h] ; r10 = nValue2 + mov r10, [rsp+028h] ; r10 = nValue2 add qword ptr [r11 + r8 * 8], r9 lea r8, [r8+1] @@ -194,9 +194,9 @@ ttmath_addvector_x64 PROC ; rdx = ss2 ; r8 = ss1_size ; r9 = ss2_size - ; [esp+0x28] = result + ; [rsp+0x28] = result - mov r10, [esp+028h] + mov r10, [rsp+028h] sub r8, r9 xor r11, r11 ; r11=0, cf=0 @@ -316,9 +316,9 @@ ttmath_subvector_x64 PROC ; rdx = ss2 ; r8 = ss1_size ; r9 = ss2_size - ; [esp+0x28] = result + ; [rsp+0x28] = result - mov r10, [esp+028h] + mov r10, [rsp+028h] sub r8, r9 xor r11, r11 ; r11=0, cf=0 From 15629c02042cdffc53b5e383296873522f1aee6a Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Tue, 8 Oct 2019 00:03:41 +0000 Subject: [PATCH 064/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 8424ed4ac5..97068a3ac1 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1622 + FORMAL BUILD NUMBER:1623 */ -#define PRODUCT_VER_STRING "4.0.0.1622" -#define FILE_VER_STRING "WI-T4.0.0.1622" -#define LICENSE_VER_STRING "WI-T4.0.0.1622" -#define FILE_VER_NUMBER 4, 0, 0, 1622 +#define PRODUCT_VER_STRING "4.0.0.1623" +#define FILE_VER_STRING "WI-T4.0.0.1623" +#define LICENSE_VER_STRING "WI-T4.0.0.1623" +#define FILE_VER_NUMBER 4, 0, 0, 1623 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1622" +#define FB_BUILD_NO "1623" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 96f2edb5b0..dfcf8e74cb 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1622 +BuildNum=1623 NowAt=`pwd` cd `dirname $0` From dd15e6ad081ddabb8c8f792b9f5d06b3276423bb Mon Sep 17 00:00:00 2001 From: Dmitry Yemanov Date: Tue, 8 Oct 2019 14:01:19 +0300 Subject: [PATCH 065/274] Fixed crash when multiple replicas are configured on the slave --- src/jrd/replication/Config.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jrd/replication/Config.cpp b/src/jrd/replication/Config.cpp index 63ba60fe96..aba689570b 100644 --- a/src/jrd/replication/Config.cpp +++ b/src/jrd/replication/Config.cpp @@ -245,8 +245,6 @@ void Config::enumerate(Firebird::Array& replicas) ConfigFile cfgFile(filename, ConfigFile::HAS_SUB_CONF | ConfigFile::NATIVE_ORDER | ConfigFile::CUSTOM_MACROS); - AutoPtr config(FB_NEW Config); - bool defaultFound = false, exactMatch = false; const ConfigFile::Parameters& params = cfgFile.getParameters(); for (const auto& section : params) @@ -256,6 +254,8 @@ void Config::enumerate(Firebird::Array& replicas) PathName dbName(section.value.c_str()); + AutoPtr config(FB_NEW Config); + const ConfigFile::Parameters& elements = section.sub->getParameters(); for (const auto& el : elements) { From ea5d78a100484b6059071da2e7799018bbc57272 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 9 Oct 2019 00:03:44 +0000 Subject: [PATCH 066/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 97068a3ac1..e4d1a29258 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1623 + FORMAL BUILD NUMBER:1624 */ -#define PRODUCT_VER_STRING "4.0.0.1623" -#define FILE_VER_STRING "WI-T4.0.0.1623" -#define LICENSE_VER_STRING "WI-T4.0.0.1623" -#define FILE_VER_NUMBER 4, 0, 0, 1623 +#define PRODUCT_VER_STRING "4.0.0.1624" +#define FILE_VER_STRING "WI-T4.0.0.1624" +#define LICENSE_VER_STRING "WI-T4.0.0.1624" +#define FILE_VER_NUMBER 4, 0, 0, 1624 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1623" +#define FB_BUILD_NO "1624" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index dfcf8e74cb..35c2ee0713 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1623 +BuildNum=1624 NowAt=`pwd` cd `dirname $0` From aa70f4f9444ed85a9dfa1c13b96f7633e0533723 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Thu, 10 Oct 2019 12:59:03 -0300 Subject: [PATCH 067/274] Fixed CORE-4739 - Accent insensitive comparison: Diacritical letters with DIAGONAL crossing stroke pass only test on EQUALITY to their non-accented forms. --- src/common/unicode_util.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/common/unicode_util.cpp b/src/common/unicode_util.cpp index af5b3ff98f..6d3df455d3 100644 --- a/src/common/unicode_util.cpp +++ b/src/common/unicode_util.cpp @@ -213,10 +213,17 @@ public: { ciAiTransCacheMutex.leave(); - UErrorCode errorCode = U_ZERO_ERROR; // Fix for CORE-4136. Was "Any-Upper; NFD; [:Nonspacing Mark:] Remove; NFC". - ret = utransOpen("NFD; [:Nonspacing Mark:] Remove; NFC", - UTRANS_FORWARD, NULL, 0, NULL, &errorCode); + // Also see CORE-4739. + static const auto RULE = + u"::NFD; ::[:Nonspacing Mark:] Remove; ::NFC;" + " \\u00d0 > D;" // LATIN CAPITAL LETTER ETH' (U+00D0), iceland + " \\u00d8 > O;" // LATIN CAPITAL LETTER O WITH STROKE' (U+00D8), used in danish & iceland alphabets; + " \\u013f > L;" // LATIN CAPITAL LETTER L WITH MIDDLE DOT' (U+013F), catalone (valencian) + " \\u0141 > L;"; // LATIN CAPITAL LETTER L WITH STROKE' (U+0141), polish + + UErrorCode errorCode = U_ZERO_ERROR; + ret = utransOpenU(u"FbNormalizer", -1, UTRANS_FORWARD, RULE, -1, NULL, &errorCode); } return ret; @@ -257,8 +264,9 @@ public: void (U_EXPORT2 *ucolGetVersion)(const UCollator* coll, UVersionInfo info); void (U_EXPORT2 *utransClose)(UTransliterator* trans); - UTransliterator* (U_EXPORT2 *utransOpen)( - const char* id, + UTransliterator* (U_EXPORT2 *utransOpenU)( + const UChar* id, + int32_t idLength, UTransDirection dir, const UChar* rules, /* may be Null */ int32_t rulesLength, /* -1 if null-terminated */ @@ -1138,7 +1146,7 @@ UnicodeUtil::ICU* UnicodeUtil::loadICU(const string& icuVersion, const string& c icu->getEntryPoint("ucol_setAttribute", icu->inModule, icu->ucolSetAttribute); icu->getEntryPoint("ucol_strcoll", icu->inModule, icu->ucolStrColl); icu->getEntryPoint("ucol_getVersion", icu->inModule, icu->ucolGetVersion); - icu->getEntryPoint("utrans_open", icu->inModule, icu->utransOpen); + icu->getEntryPoint("utrans_openU", icu->inModule, icu->utransOpenU); icu->getEntryPoint("utrans_close", icu->inModule, icu->utransClose); icu->getEntryPoint("utrans_transUChars", icu->inModule, icu->utransTransUChars); } From 374536e8175e9a5e3585e4de534b3f3e0ad7e824 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Thu, 10 Oct 2019 13:15:11 -0300 Subject: [PATCH 068/274] Try to fix travis build. --- src/common/unicode_util.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/unicode_util.cpp b/src/common/unicode_util.cpp index 6d3df455d3..5bec7edfc3 100644 --- a/src/common/unicode_util.cpp +++ b/src/common/unicode_util.cpp @@ -215,7 +215,7 @@ public: // Fix for CORE-4136. Was "Any-Upper; NFD; [:Nonspacing Mark:] Remove; NFC". // Also see CORE-4739. - static const auto RULE = + static const auto RULE = (const UChar*) u"::NFD; ::[:Nonspacing Mark:] Remove; ::NFC;" " \\u00d0 > D;" // LATIN CAPITAL LETTER ETH' (U+00D0), iceland " \\u00d8 > O;" // LATIN CAPITAL LETTER O WITH STROKE' (U+00D8), used in danish & iceland alphabets; From d7f323a063f9727906c8680c9fb0b5a75bfa3b96 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Thu, 10 Oct 2019 13:25:08 -0300 Subject: [PATCH 069/274] Try to fix travis build. --- src/common/unicode_util.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/unicode_util.cpp b/src/common/unicode_util.cpp index 5bec7edfc3..d35082f781 100644 --- a/src/common/unicode_util.cpp +++ b/src/common/unicode_util.cpp @@ -223,7 +223,7 @@ public: " \\u0141 > L;"; // LATIN CAPITAL LETTER L WITH STROKE' (U+0141), polish UErrorCode errorCode = U_ZERO_ERROR; - ret = utransOpenU(u"FbNormalizer", -1, UTRANS_FORWARD, RULE, -1, NULL, &errorCode); + ret = utransOpenU((const UChar*) u"FbNormalizer", -1, UTRANS_FORWARD, RULE, -1, NULL, &errorCode); } return ret; From 3470ec0932d828879e880d2db772ed957cf67ad3 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Fri, 11 Oct 2019 00:03:44 +0000 Subject: [PATCH 070/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index e4d1a29258..c361df9148 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1624 + FORMAL BUILD NUMBER:1627 */ -#define PRODUCT_VER_STRING "4.0.0.1624" -#define FILE_VER_STRING "WI-T4.0.0.1624" -#define LICENSE_VER_STRING "WI-T4.0.0.1624" -#define FILE_VER_NUMBER 4, 0, 0, 1624 +#define PRODUCT_VER_STRING "4.0.0.1627" +#define FILE_VER_STRING "WI-T4.0.0.1627" +#define LICENSE_VER_STRING "WI-T4.0.0.1627" +#define FILE_VER_NUMBER 4, 0, 0, 1627 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1624" +#define FB_BUILD_NO "1627" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 35c2ee0713..5fedcaaeef 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1624 +BuildNum=1627 NowAt=`pwd` cd `dirname $0` From 69ba7fc920fc69e123b4b41416033cc47d0598af Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 11 Oct 2019 19:58:47 +0300 Subject: [PATCH 071/274] Fixed CORE-6000: gbak issues "Your user name and password are not defined" when command switch "-fe(tch_password) ..." is specified when run as service --- src/burp/burp.cpp | 70 +++++++++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/src/burp/burp.cpp b/src/burp/burp.cpp index d2b83167c0..2b863a9b9e 100644 --- a/src/burp/burp.cpp +++ b/src/burp/burp.cpp @@ -116,6 +116,8 @@ static int svc_api_gbak(Firebird::UtilSvc*, const Switches& switches); static void burp_output(bool err, const SCHAR*, ...) ATTRIBUTE_FORMAT(2,3); static void burp_usage(const Switches& switches); static Switches::in_sw_tab_t* findSwitchOrThrow(Firebird::UtilSvc*, Switches& switches, Firebird::string& sw); +static void processFetchPass(const SCHAR*& password, int& itr, const int argc, Firebird::UtilSvc::ArgvType& argv); + // fil.fil_length is FB_UINT64 const ULONG KBYTE = 1024; @@ -182,6 +184,7 @@ static int svc_api_gbak(Firebird::UtilSvc* uSvc, const Switches& switches) * **********************************************/ Firebird::string usr, pswd, service; + const SCHAR* pswd2 = NULL; bool flag_restore = false; bool flag_verbose = false; #ifdef TRUSTED_AUTH @@ -235,6 +238,7 @@ static int svc_api_gbak(Firebird::UtilSvc* uSvc, const Switches& switches) break; case IN_SW_BURP_PASS: // default password pswd = argv[itr]; + pswd2 = pswd.nullStr(); uSvc->hidePasswd(argv, itr); break; case IN_SW_BURP_SE: // service name @@ -244,6 +248,12 @@ static int svc_api_gbak(Firebird::UtilSvc* uSvc, const Switches& switches) argv[itr] = 0; } break; + case IN_SW_BURP_FETCHPASS: + argv[itr] = 0; + processFetchPass(pswd2, itr, argc, argv); + pswd = pswd2; + argv[itr] = 0; + break; case IN_SW_BURP_V: // verify actions if (flag_verbint) BURP_error(329, true); // verify (verbose) and verbint options are mutually exclusive @@ -675,33 +685,7 @@ int gbak(Firebird::UtilSvc* uSvc) tdgbl->gbl_sw_password = argv[itr]; break; case IN_SW_BURP_FETCHPASS: - if (++itr >= argc) - { - BURP_error(189, true); - // password parameter missing - } - if (tdgbl->gbl_sw_password) - { - BURP_error(307, true); - // too many passwords provided - } - switch (fb_utils::fetchPassword(argv[itr], tdgbl->gbl_sw_password)) - { - case fb_utils::FETCH_PASS_OK: - break; - case fb_utils::FETCH_PASS_FILE_OPEN_ERROR: - BURP_error(308, true, MsgFormat::SafeArg() << argv[itr] << errno); - // error @2 opening password file @1 - break; - case fb_utils::FETCH_PASS_FILE_READ_ERROR: - BURP_error(309, true, MsgFormat::SafeArg() << argv[itr] << errno); - // error @2 reading password file @1 - break; - case fb_utils::FETCH_PASS_FILE_EMPTY: - BURP_error(310, true, MsgFormat::SafeArg() << argv[itr]); - // password file @1 is empty - break; - } + processFetchPass(tdgbl->gbl_sw_password, itr, argc, argv); break; case IN_SW_BURP_USER: if (++itr >= argc) @@ -2710,3 +2694,35 @@ void BURP_makeSymbol(BurpGlobals* tdgbl, Firebird::string& name) // add double name.insert(0u, 1, dq); name += dq; } + +static void processFetchPass(const SCHAR*& password, int& itr, const int argc, Firebird::UtilSvc::ArgvType& argv) +{ + if (++itr >= argc) + { + BURP_error(189, true); + // password parameter missing + } + if (password) + { + BURP_error(307, true); + // too many passwords provided + } + + switch (fb_utils::fetchPassword(argv[itr], password)) + { + case fb_utils::FETCH_PASS_OK: + break; + case fb_utils::FETCH_PASS_FILE_OPEN_ERROR: + BURP_error(308, true, MsgFormat::SafeArg() << argv[itr] << errno); + // error @2 opening password file @1 + break; + case fb_utils::FETCH_PASS_FILE_READ_ERROR: + BURP_error(309, true, MsgFormat::SafeArg() << argv[itr] << errno); + // error @2 reading password file @1 + break; + case fb_utils::FETCH_PASS_FILE_EMPTY: + BURP_error(310, true, MsgFormat::SafeArg() << argv[itr]); + // password file @1 is empty + break; + } +} \ No newline at end of file From 56bf40b22aca7fb6df74852c8c120474f05b9108 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Fri, 11 Oct 2019 12:40:30 -0300 Subject: [PATCH 072/274] Fix VS 2015 AppVeyor build. --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index fa9564dd4c..ec259323de 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -32,6 +32,7 @@ install: - cmd: if "%PLATFORM%" == "x86" set FB_OUTPUT_SUFFIX=win32 - cmd: if "%PLATFORM%" == "x86" set FB_VS_ARCH=x86 - cmd: if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" call "%ProgramFiles(x86)%\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=%FB_VS_ARCH% + - cmd: if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2015" call "%ProgramFiles(x86)%\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %FB_VS_ARCH% - cmd: cd builds\win32 - cmd: run_all.bat JUSTBUILD - cmd: set ARTIFACTS_PATH=output_%FB_OUTPUT_SUFFIX% From 21b847864d44a282c8c149b2cebfeb3ecae15c53 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 12 Oct 2019 00:03:52 +0000 Subject: [PATCH 073/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index c361df9148..c8a062d51f 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1627 + FORMAL BUILD NUMBER:1629 */ -#define PRODUCT_VER_STRING "4.0.0.1627" -#define FILE_VER_STRING "WI-T4.0.0.1627" -#define LICENSE_VER_STRING "WI-T4.0.0.1627" -#define FILE_VER_NUMBER 4, 0, 0, 1627 +#define PRODUCT_VER_STRING "4.0.0.1629" +#define FILE_VER_STRING "WI-T4.0.0.1629" +#define LICENSE_VER_STRING "WI-T4.0.0.1629" +#define FILE_VER_NUMBER 4, 0, 0, 1629 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1627" +#define FB_BUILD_NO "1629" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 5fedcaaeef..c0f600981a 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1627 +BuildNum=1629 NowAt=`pwd` cd `dirname $0` From 6203f07cccad5b5a40c508475d5dc9c4bcc9a82d Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Wed, 16 Oct 2019 21:36:14 -0300 Subject: [PATCH 074/274] Fixed CORE-6158 - substring similar - extra characters in the result for non latin characters. --- src/dsql/ExprNodes.cpp | 11 +++++------ src/jrd/Collation.cpp | 13 ++++++++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/dsql/ExprNodes.cpp b/src/dsql/ExprNodes.cpp index 9648041274..b8a5a23304 100644 --- a/src/dsql/ExprNodes.cpp +++ b/src/dsql/ExprNodes.cpp @@ -11895,18 +11895,17 @@ dsc* SubstringSimilarNode::execute(thread_db* tdbb, jrd_req* request) const if (evaluator->result()) { - // Get the byte bounds of the matched substring. - unsigned start = 0; - unsigned length = 0; + // Get the character bounds of the matched substring. + unsigned start, length; evaluator->getResultInfo(&start, &length); dsc desc; desc.makeText((USHORT) exprLen, textType); EVL_make_value(tdbb, &desc, impure); - // And return it. - memcpy(impure->vlu_desc.dsc_address, exprStr + start, length); - impure->vlu_desc.dsc_length = length; + impure->vlu_desc.dsc_length = charSet->substring(exprLen, exprStr, + impure->vlu_desc.dsc_length, impure->vlu_desc.dsc_address, + start, length); return &impure->vlu_desc; } diff --git a/src/jrd/Collation.cpp b/src/jrd/Collation.cpp index 33f8a14f1e..c535dcca34 100644 --- a/src/jrd/Collation.cpp +++ b/src/jrd/Collation.cpp @@ -311,7 +311,18 @@ public: if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE) UnicodeUtil::utf8Normalize(*bufferPtr); - return regex->matches((const char*) bufferPtr->begin(), bufferPtr->getCount(), &resultStart, &resultLength); + if (!regex->matches((const char*) bufferPtr->begin(), bufferPtr->getCount(), &resultStart, &resultLength)) + return false; + + if (charSetId != CS_NONE && charSetId != CS_BINARY) + { + // Get the character positions in the utf-8 string. + auto utf8CharSet = IntlUtil::getUtf8CharSet(); + resultLength = utf8CharSet->length(resultLength, bufferPtr->begin() + resultStart, true); + resultStart = utf8CharSet->length(resultStart, bufferPtr->begin(), true); + } + + return true; } virtual void getResultInfo(unsigned* start, unsigned* length) From 329a5c08ce4c2dda7a83d4b35f61e2474eab5ab4 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Thu, 17 Oct 2019 13:03:23 -0300 Subject: [PATCH 075/274] Fixed CORE-6159 and CORE-6160: CORE-6159 - SUBSTRING SIMILAR is described with wrong data type in DSQL. CORE-6160 - SUBSTRING of non-text/-blob is described to return NONE character set in DSQL. --- src/dsql/ExprNodes.cpp | 7 ++++++- src/jrd/DataTypeUtil.cpp | 8 +++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/dsql/ExprNodes.cpp b/src/dsql/ExprNodes.cpp index b8a5a23304..a8fee2360a 100644 --- a/src/dsql/ExprNodes.cpp +++ b/src/dsql/ExprNodes.cpp @@ -11761,7 +11761,12 @@ void SubstringSimilarNode::genBlr(DsqlCompilerScratch* dsqlScratch) void SubstringSimilarNode::make(DsqlCompilerScratch* dsqlScratch, dsc* desc) { - DsqlDescMaker::fromNode(dsqlScratch, desc, expr, true); + dsc exprDesc; + DsqlDescMaker::fromNode(dsqlScratch, &exprDesc, expr); + + DSqlDataTypeUtil dataTypeUtil(dsqlScratch); + dataTypeUtil.makeSubstr(desc, &exprDesc, nullptr, nullptr); + desc->setNullable(true); } void SubstringSimilarNode::getDesc(thread_db* tdbb, CompilerScratch* csb, dsc* desc) diff --git a/src/jrd/DataTypeUtil.cpp b/src/jrd/DataTypeUtil.cpp index 35fde99704..5f4a83c174 100644 --- a/src/jrd/DataTypeUtil.cpp +++ b/src/jrd/DataTypeUtil.cpp @@ -305,14 +305,16 @@ void DataTypeUtilBase::makeSubstr(dsc* result, const dsc* value, const dsc* offs result->dsc_dtype = dtype_varying; } - result->setTextType(value->getTextType()); - result->setNullable(value->isNullable() || offset->isNullable() || length->isNullable()); + result->setTextType(value->isText() ? value->getTextType() : CS_ASCII); + result->setNullable(value->isNullable() || + (offset && offset->isNullable()) || + (length && length->isNullable())); if (result->isText()) { ULONG len = convertLength(value, result); - if (length->dsc_address) // constant + if (length && length->dsc_address) // constant { SLONG constant = CVT_get_long(length, 0, JRD_get_thread_data()->getAttachment()->att_dec_status, ERR_post); fb_assert(constant >= 0); From 83a3c33e5001c4c57935539b7eccbddaa2256c42 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Fri, 18 Oct 2019 00:03:52 +0000 Subject: [PATCH 076/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index c8a062d51f..f704add079 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1629 + FORMAL BUILD NUMBER:1631 */ -#define PRODUCT_VER_STRING "4.0.0.1629" -#define FILE_VER_STRING "WI-T4.0.0.1629" -#define LICENSE_VER_STRING "WI-T4.0.0.1629" -#define FILE_VER_NUMBER 4, 0, 0, 1629 +#define PRODUCT_VER_STRING "4.0.0.1631" +#define FILE_VER_STRING "WI-T4.0.0.1631" +#define LICENSE_VER_STRING "WI-T4.0.0.1631" +#define FILE_VER_NUMBER 4, 0, 0, 1631 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1629" +#define FB_BUILD_NO "1631" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index c0f600981a..6e97dc2386 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1629 +BuildNum=1631 NowAt=`pwd` cd `dirname $0` From 327c642f10ca9d2a94a7fb0f0ed68999537dd56a Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Fri, 18 Oct 2019 12:40:43 -0300 Subject: [PATCH 077/274] Postfix for CORE-6160. --- src/jrd/DataTypeUtil.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jrd/DataTypeUtil.cpp b/src/jrd/DataTypeUtil.cpp index 5f4a83c174..ea556a41d2 100644 --- a/src/jrd/DataTypeUtil.cpp +++ b/src/jrd/DataTypeUtil.cpp @@ -305,7 +305,7 @@ void DataTypeUtilBase::makeSubstr(dsc* result, const dsc* value, const dsc* offs result->dsc_dtype = dtype_varying; } - result->setTextType(value->isText() ? value->getTextType() : CS_ASCII); + result->setTextType(value->isText() || value->isBlob() ? value->getTextType() : CS_ASCII); result->setNullable(value->isNullable() || (offset && offset->isNullable()) || (length && length->isNullable())); From c82a119be6dcc545c1f4b95564571eeea83b70ef Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 19 Oct 2019 00:03:54 +0000 Subject: [PATCH 078/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index f704add079..b12bd0d0c1 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1631 + FORMAL BUILD NUMBER:1632 */ -#define PRODUCT_VER_STRING "4.0.0.1631" -#define FILE_VER_STRING "WI-T4.0.0.1631" -#define LICENSE_VER_STRING "WI-T4.0.0.1631" -#define FILE_VER_NUMBER 4, 0, 0, 1631 +#define PRODUCT_VER_STRING "4.0.0.1632" +#define FILE_VER_STRING "WI-T4.0.0.1632" +#define LICENSE_VER_STRING "WI-T4.0.0.1632" +#define FILE_VER_NUMBER 4, 0, 0, 1632 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1631" +#define FB_BUILD_NO "1632" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 6e97dc2386..68bfc76ffb 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1631 +BuildNum=1632 NowAt=`pwd` cd `dirname $0` From 80245dd81df9ad3988056d0940ff0280d1c0ffae Mon Sep 17 00:00:00 2001 From: Alex Peshkoff Date: Sat, 19 Oct 2019 14:48:46 +0200 Subject: [PATCH 079/274] Fixed CORE-6163: Generator pages are not encrypted --- src/jrd/ods.h | 2 +- src/msgs/facilities2.sql | 2 +- src/msgs/messages2.sql | 1 + src/utilities/gstat/dba.epp | 6 +++++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/jrd/ods.h b/src/jrd/ods.h index 9de23b4c95..433944e9e9 100644 --- a/src/jrd/ods.h +++ b/src/jrd/ods.h @@ -216,7 +216,7 @@ namespace Ods { const bool pag_crypt_page[pag_max + 1] = {false, false, false, false, false, true, // data false, true, true, // index, blob - false, false}; + true, false}; // generators // pag_flags for any page type diff --git a/src/msgs/facilities2.sql b/src/msgs/facilities2.sql index 67134df131..bcf03c80dd 100644 --- a/src/msgs/facilities2.sql +++ b/src/msgs/facilities2.sql @@ -15,7 +15,7 @@ set bulk_insert INSERT INTO FACILITIES (LAST_CHANGE, FACILITY, FAC_CODE, MAX_NUM ('2018-02-27 14:50:31', 'JRD_BUGCHK', 15, 308) ('2016-05-26 13:53:45', 'ISQL', 17, 197) ('2010-07-10 10:50:30', 'GSEC', 18, 105) -('2018-04-18 18:52:29', 'GSTAT', 21, 62) +('2019-10-19 12:52:29', 'GSTAT', 21, 63) ('2013-12-19 17:31:31', 'FBSVCMGR', 22, 58) ('2009-07-18 12:12:12', 'UTL', 23, 2) ('2016-03-20 15:30:00', 'NBACKUP', 24, 80) diff --git a/src/msgs/messages2.sql b/src/msgs/messages2.sql index 4757c50a63..5963594b59 100644 --- a/src/msgs/messages2.sql +++ b/src/msgs/messages2.sql @@ -3420,6 +3420,7 @@ Analyzing database pages ...', NULL, NULL); (NULL, 'main', 'dba.epp', NULL, 21, 59, NULL, 'Gstat execution time @1', NULL, NULL) (NULL, 'main', 'dba.epp', NULL, 21, 60, NULL, 'Gstat completion time @1', NULL, NULL) (NULL, 'lastUsedPage', 'dba.epp', NULL, 21, 61, NULL, ' Expected page inventory page @1', NULL, NULL); +(NULL, 'main', 'dba.epp', NULL, 21, 62, NULL, 'Generator pages: total @1, encrypted @2, non-crypted @3', NULL, NULL) -- FBSVCMGR -- All messages use the new format. ('fbsvcmgr_bad_am', 'putAccessMode', 'fbsvcmgr.cpp', NULL, 22, 1, NULL, 'Wrong value for access mode', NULL, NULL); diff --git a/src/utilities/gstat/dba.epp b/src/utilities/gstat/dba.epp index 23823c9cc3..b38f9634a6 100644 --- a/src/utilities/gstat/dba.epp +++ b/src/utilities/gstat/dba.epp @@ -715,7 +715,7 @@ int gstat(Firebird::UtilSvc* uSvc) private: ULONG enc, non; }; - Statist data, index, blob, other; + Statist data, index, blob, generator, other; ULONG last = lastUsedPage(header->hdr_page_size); for (page = 0; page <= last; ++page) @@ -737,6 +737,9 @@ int gstat(Firebird::UtilSvc* uSvc) case pag_blob: blob.log(p->pag_flags); break; + case pag_ids: + generator.log(p->pag_flags); + break; default: other.log(p->pag_flags); break; @@ -747,6 +750,7 @@ int gstat(Firebird::UtilSvc* uSvc) data.print(52); index.print(53); blob.print(54); + generator.print(62); if (other.hasCrypted()) other.print(58); From 0b62ee93ff4310ac2575c260889fc24245d4a850 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sun, 20 Oct 2019 00:03:55 +0000 Subject: [PATCH 080/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index b12bd0d0c1..88f804d728 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1632 + FORMAL BUILD NUMBER:1633 */ -#define PRODUCT_VER_STRING "4.0.0.1632" -#define FILE_VER_STRING "WI-T4.0.0.1632" -#define LICENSE_VER_STRING "WI-T4.0.0.1632" -#define FILE_VER_NUMBER 4, 0, 0, 1632 +#define PRODUCT_VER_STRING "4.0.0.1633" +#define FILE_VER_STRING "WI-T4.0.0.1633" +#define LICENSE_VER_STRING "WI-T4.0.0.1633" +#define FILE_VER_NUMBER 4, 0, 0, 1633 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1632" +#define FB_BUILD_NO "1633" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 68bfc76ffb..d6bf17b0d9 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1632 +BuildNum=1633 NowAt=`pwd` cd `dirname $0` From 760d4b13ca67a3619f7aa9bda2d783012ac3bda5 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Mon, 21 Oct 2019 12:53:13 -0300 Subject: [PATCH 081/274] Fixed CORE-6166 - Problems with long object names (> 255 bytes). --- src/common/classes/QualifiedName.h | 5 +++ src/dsql/metd.epp | 16 +++++-- src/isql/extract.epp | 14 +++---- src/isql/isql.epp | 8 ++-- src/isql/show.epp | 26 ++++++------ src/jrd/Attachment.h | 14 ++++++- src/jrd/met.epp | 67 +++++++++++++++++++----------- src/jrd/met_proto.h | 4 +- 8 files changed, 99 insertions(+), 55 deletions(-) diff --git a/src/common/classes/QualifiedName.h b/src/common/classes/QualifiedName.h index 6557ad18f1..f1fa317bb5 100644 --- a/src/common/classes/QualifiedName.h +++ b/src/common/classes/QualifiedName.h @@ -85,6 +85,11 @@ public: return identifier == m.identifier && package == m.package; } + bool operator !=(const QualifiedName& m) const + { + return !(identifier == m.identifier && package == m.package); + } + public: string toString() const { diff --git a/src/dsql/metd.epp b/src/dsql/metd.epp index a338643359..9eb1ea8528 100644 --- a/src/dsql/metd.epp +++ b/src/dsql/metd.epp @@ -637,10 +637,14 @@ dsql_udf* METD_get_function(jrd_tra* transaction, DsqlCompilerScratch* dsqlScrat if (MET_dsql_cache_use(tdbb, SYM_udf, metaName.identifier, metaName.package)) userFunc->udf_flags |= UDF_dropped; - - return userFunc; } + if (userFunc && (userFunc->udf_flags & UDF_dropped)) + userFunc = nullptr; + + if (userFunc) + return userFunc; + // Now see if it is in the database USHORT return_arg = 0; @@ -995,10 +999,14 @@ dsql_prc* METD_get_procedure(jrd_tra* transaction, DsqlCompilerScratch* dsqlScra if (MET_dsql_cache_use(tdbb, SYM_procedure, metaName.identifier, metaName.package)) procedure->prc_flags |= PRC_dropped; - - return procedure; } + if (procedure && (procedure->prc_flags & PRC_dropped)) + procedure = nullptr; + + if (procedure) + return procedure; + // now see if it is in the database while (!procedure) diff --git a/src/isql/extract.epp b/src/isql/extract.epp index d817f09752..33ecd7a499 100644 --- a/src/isql/extract.epp +++ b/src/isql/extract.epp @@ -97,7 +97,7 @@ static void list_views(); static const char* const Procterm = "^"; // TXNN: script use only -static TEXT SQL_identifier[BUFFER_LENGTH128]; +static TEXT SQL_identifier[BUFFER_LENGTH256]; static TEXT SQL_identifier2[BUFFER_XLARGE]; // should be >= 1026 bytes (exception text + two quotes) @@ -727,8 +727,8 @@ static void get_procedure_args(const char* proc_name) if (!PRM.RDB$PARAMETER_MECHANISM.NULL) prm_mech = (prm_mech_t) PRM.RDB$PARAMETER_MECHANISM; - char relationName[BUFFER_LENGTH128] = ""; - char relationField[BUFFER_LENGTH128] = ""; + char relationName[BUFFER_LENGTH256] = ""; + char relationField[BUFFER_LENGTH256] = ""; if (!PRM.RDB$RELATION_NAME.NULL) { @@ -942,8 +942,8 @@ static void get_function_args_ods12(const char* func_name, USHORT out_arg) if (!ARG.RDB$ARGUMENT_MECHANISM.NULL) prm_mech = (prm_mech_t) ARG.RDB$ARGUMENT_MECHANISM; - char relationName[BUFFER_LENGTH128] = ""; - char relationField[BUFFER_LENGTH128] = ""; + char relationName[BUFFER_LENGTH256] = ""; + char relationField[BUFFER_LENGTH256] = ""; if (!ARG.RDB$RELATION_NAME.NULL) { @@ -2905,8 +2905,8 @@ static void list_functions_legacy() * RETURNS INTEGER BY VALUE * ENTRY_POINT entrypoint MODULE_NAME module; **************************************/ - char type_buffer[BUFFER_LENGTH128]; - char return_buffer[BUFFER_LENGTH128]; + char type_buffer[BUFFER_LENGTH256]; + char return_buffer[BUFFER_LENGTH256]; bool first = true; diff --git a/src/isql/isql.epp b/src/isql/isql.epp index 5a73a6a30f..033194a87f 100644 --- a/src/isql/isql.epp +++ b/src/isql/isql.epp @@ -1441,7 +1441,7 @@ SLONG ISQL_get_index_segments(TEXT* segs, * returns the list of columns in an index. * **************************************/ - TEXT SQL_identifier[BUFFER_LENGTH128]; + TEXT SQL_identifier[BUFFER_LENGTH256]; *segs = '\0'; @@ -6222,8 +6222,8 @@ static processing_state newdb(TEXT* dbname, strcpy(dbname, save_database); ISQL_FREE(save_database); - TEXT local_psw[BUFFER_LENGTH128]; - TEXT local_usr[BUFFER_LENGTH128]; + TEXT local_psw[BUFFER_LENGTH256]; + TEXT local_usr[BUFFER_LENGTH256]; TEXT local_sql_role[BUFFER_LENGTH256]; // global user and passwords are set only if they are not originally set @@ -8303,6 +8303,8 @@ static unsigned process_message_display(Firebird::IMessageMetadata* message, uns if (charSet == CS_UTF8) disp_length *= 4; } + else if (isqlGlob.att_charset == CS_UTF8) + disp_length = MAX(disp_length, namelength * 4); // The total line length linelength += disp_length + 1; diff --git a/src/isql/show.epp b/src/isql/show.epp index 77eced9e6b..b36ebeb6ab 100644 --- a/src/isql/show.epp +++ b/src/isql/show.epp @@ -113,7 +113,7 @@ static processing_state show_users12(); const char* const spaces = " "; static TEXT Print_buffer[512]; -static TEXT SQL_identifier[BUFFER_LENGTH128]; +static TEXT SQL_identifier[BUFFER_LENGTH256]; static bool reReadDbOwner = true; @@ -764,7 +764,7 @@ processing_state SHOW_grants2 (const SCHAR* object, prev_user_type = -1; prev_object_type = -1; char priv_string[MAX_PRIV_LIST] = ""; - char col_string[BUFFER_LENGTH128] = ""; + char col_string[BUFFER_LENGTH256] = ""; char with_option[19] = ""; USHORT priv_flags = 0; SSHORT prev_field_null = -1; @@ -1029,7 +1029,7 @@ processing_state SHOW_grants2 (const SCHAR* object, } // No procedure called "object" was found, try role "object" - SCHAR role_name[BUFFER_LENGTH128]; + SCHAR role_name[BUFFER_LENGTH256]; if (obj_type == obj_sql_role || obj_type == 255) { @@ -1835,7 +1835,7 @@ void SHOW_grant_roles2 (const SCHAR* terminator, * All membership privilege may have the with_admin or/and default options set. * **************************************/ - TEXT SQL_identifier2[BUFFER_LENGTH128]; + TEXT SQL_identifier2[BUFFER_LENGTH256]; BASED_ON RDB$USER_PRIVILEGES.RDB$GRANTOR dummy; // used to declare buf_grantor FB_UNUSED_VAR(dummy); // Silence compiler warning about unused variable SCHAR buf_grantor[sizeof(dummy) + 20]; @@ -2099,7 +2099,7 @@ processing_state SHOW_metadata(const SCHAR* const* cmd, SCHAR** lcmd) if (ret == ps_ERR || handled) return ret; - TEXT SQL_id_for_grant[BUFFER_LENGTH128]; + TEXT SQL_id_for_grant[BUFFER_LENGTH256]; int key = 0; switch (showoptions.getCommand(cmd[1])) @@ -3040,8 +3040,8 @@ static void show_comment(const char* objtype, char* packageName, char* name1, ch if (name2) fb_utils::exact_name(name2); - char packageNameBuffer[BUFFER_LENGTH128]; - char SQL_identifier2[BUFFER_LENGTH128]; + char packageNameBuffer[BUFFER_LENGTH256]; + char SQL_identifier2[BUFFER_LENGTH256]; if (escape_quotes && isqlGlob.db_SQL_dialect > SQL_DIALECT_V6_TRANSITION) { @@ -4324,8 +4324,8 @@ static processing_state show_func(const SCHAR* funcname) prm_default_source = ARG.RDB$DEFAULT_SOURCE; } - char relationName[BUFFER_LENGTH128] = ""; - char relationField[BUFFER_LENGTH128] = ""; + char relationName[BUFFER_LENGTH256] = ""; + char relationField[BUFFER_LENGTH256] = ""; if (!ARG.RDB$RELATION_NAME.NULL) { @@ -4888,7 +4888,7 @@ static processing_state show_packages(const SCHAR* package_name) static void printIdent(bool quote, char* ident, const char* format = NULL) { fb_utils::exact_name(ident); - char quotedIdent[BUFFER_LENGTH128]; + char quotedIdent[BUFFER_LENGTH256]; if (quote && isqlGlob.db_SQL_dialect > SQL_DIALECT_V6_TRANSITION) { IUTILS_copy_SQL_id(ident, quotedIdent, DBL_QUOTE); @@ -5205,8 +5205,8 @@ static processing_state show_proc(const SCHAR* procname) if (!prm_collation_null) collation = PRM.RDB$COLLATION_ID; - char relationName[BUFFER_LENGTH128] = ""; - char relationField[BUFFER_LENGTH128] = ""; + char relationName[BUFFER_LENGTH256] = ""; + char relationField[BUFFER_LENGTH256] = ""; if (!PRM.RDB$RELATION_NAME.NULL) { @@ -5426,7 +5426,7 @@ static processing_state show_role(const SCHAR* object, bool system, const char* } // show role with role supplied, display users and other roles granted this role - SCHAR role_name[BUFFER_LENGTH128]; + SCHAR role_name[BUFFER_LENGTH256]; bool first = true; FOR FIRST 1 R IN RDB$ROLES WITH R.RDB$ROLE_NAME EQ object diff --git a/src/jrd/Attachment.h b/src/jrd/Attachment.h index b4c47f434e..46c3229b70 100644 --- a/src/jrd/Attachment.h +++ b/src/jrd/Attachment.h @@ -35,6 +35,7 @@ #include "../common/classes/ByteChunk.h" #include "../common/classes/GenericMap.h" +#include "../common/classes/QualifiedName.h" #include "../common/classes/SyncObject.h" #include "../common/classes/array.h" #include "../common/classes/stack.h" @@ -93,12 +94,21 @@ namespace Jrd struct DSqlCacheItem { + DSqlCacheItem(MemoryPool& pool) + : key(pool), + obsoleteMap(pool), + lock(nullptr), + locked(false) + { + } + + Firebird::string key; + Firebird::GenericMap > > obsoleteMap; Lock* lock; bool locked; - bool obsolete; }; -typedef Firebird::GenericMap > > DSqlCache; diff --git a/src/jrd/met.epp b/src/jrd/met.epp index 4cf83886a0..bc63deb7fb 100644 --- a/src/jrd/met.epp +++ b/src/jrd/met.epp @@ -90,6 +90,7 @@ #include "../jrd/PreparedStatement.h" #include "../jrd/RecordSourceNodes.h" #include "../jrd/DebugInterface.h" +#include "../common/classes/Hash.h" #include "../common/classes/MsgPrint.h" #include "../jrd/Function.h" @@ -107,7 +108,7 @@ using namespace Jrd; using namespace Firebird; static int blocking_ast_dsql_cache(void* ast_object); -static DSqlCacheItem* get_dsql_cache_item(thread_db* tdbb, int type, const QualifiedName& name); +static DSqlCacheItem* get_dsql_cache_item(thread_db* tdbb, sym_type type, const QualifiedName& name); static int blocking_ast_procedure(void*); static int blocking_ast_relation(void*); static int partners_ast_relation(void*); @@ -1353,11 +1354,13 @@ void MET_delete_shadow(thread_db* tdbb, USHORT shadow_number) } -bool MET_dsql_cache_use(thread_db* tdbb, int type, const MetaName& name, const MetaName& package) +bool MET_dsql_cache_use(thread_db* tdbb, sym_type type, const MetaName& name, const MetaName& package) { - DSqlCacheItem* item = get_dsql_cache_item(tdbb, type, QualifiedName(name, package)); + const QualifiedName qualifiedName(name, package); + DSqlCacheItem* item = get_dsql_cache_item(tdbb, type, qualifiedName); - const bool obsolete = item->obsolete; + bool obsolete = false; + item->obsoleteMap.get(qualifiedName, obsolete); if (!item->locked) { @@ -1366,30 +1369,33 @@ bool MET_dsql_cache_use(thread_db* tdbb, int type, const MetaName& name, const M item->locked = true; } - item->obsolete = false; + item->obsoleteMap.put(qualifiedName, false); return obsolete; } -void MET_dsql_cache_release(thread_db* tdbb, int type, const MetaName& name, const MetaName& package) +void MET_dsql_cache_release(thread_db* tdbb, sym_type type, const MetaName& name, const MetaName& package) { - DSqlCacheItem* item = get_dsql_cache_item(tdbb, type, QualifiedName(name, package)); + const QualifiedName qualifiedName(name, package); + DSqlCacheItem* item = get_dsql_cache_item(tdbb, type, qualifiedName); - // release lock + // release the shared lock LCK_release(tdbb, item->lock); // notify others through AST to mark as obsolete - const USHORT key_length = item->lock->lck_length; - AutoPtr temp_lock(FB_NEW_RPT(*tdbb->getDefaultPool(), key_length) - Lock(tdbb, key_length, LCK_dsql_cache)); - memcpy(temp_lock->getKeyPtr(), item->lock->getKeyPtr(), key_length); + AutoPtr tempExLock(FB_NEW_RPT(*tdbb->getDefaultPool(), item->key.length()) + Lock(tdbb, item->key.length(), LCK_dsql_cache)); + memcpy(tempExLock->getKeyPtr(), item->key.c_str(), item->key.length()); - if (LCK_lock(tdbb, temp_lock, LCK_EX, LCK_WAIT)) - LCK_release(tdbb, temp_lock); + if (LCK_lock(tdbb, tempExLock, LCK_EX, LCK_WAIT)) + LCK_release(tdbb, tempExLock); item->locked = false; - item->obsolete = false; + + GenericMap > >::Accessor accessor(&item->obsoleteMap); + for (bool found = accessor.getFirst(); found; found = accessor.getNext()) + accessor.current()->second = accessor.current()->first != qualifiedName; } @@ -4252,7 +4258,10 @@ static int blocking_ast_dsql_cache(void* ast_object) AsyncContextHolder tdbb(dbb, FB_FUNCTION, item->lock); - item->obsolete = true; + GenericMap > >::Accessor accessor(&item->obsoleteMap); + for (bool found = accessor.getFirst(); found; found = accessor.getNext()) + accessor.current()->second = true; + item->locked = false; LCK_release(tdbb, item->lock); } @@ -4263,33 +4272,43 @@ static int blocking_ast_dsql_cache(void* ast_object) } -static DSqlCacheItem* get_dsql_cache_item(thread_db* tdbb, int type, const QualifiedName& name) +static DSqlCacheItem* get_dsql_cache_item(thread_db* tdbb, sym_type type, const QualifiedName& name) { Database* dbb = tdbb->getDatabase(); Attachment* attachment = tdbb->getAttachment(); - string key((char*) &type, sizeof(type)); - int len = name.identifier.length(); + fb_assert((int) type <= MAX_UCHAR); + UCHAR ucharType = (UCHAR) type; + + string key("0"); // name (not hash) + key.append((char*) &ucharType, 1); + + USHORT len = (USHORT) name.identifier.length(); key.append((char*) &len, sizeof(len)); key.append(name.identifier.c_str(), len); - len = name.package.length(); + len = (USHORT) name.package.length(); key.append((char*) &len, sizeof(len)); key.append(name.package.c_str(), len); + if (key.length() > MAX_UCHAR) + { + FB_SIZE_T hash = DefaultHash::hash(key.c_str(), key.length(), sizeof(FB_SIZE_T)); + key = "1"; // hash + key.append((char*) &ucharType, 1); + key.append((char*) &hash, sizeof(hash)); + } + DSqlCacheItem* item = attachment->att_dsql_cache.put(key); if (item) { - item->obsolete = false; - item->locked = false; + item->key = key; item->lock = FB_NEW_RPT(*attachment->att_pool, key.length()) Lock(tdbb, key.length(), LCK_dsql_cache, item, blocking_ast_dsql_cache); memcpy(item->lock->getKeyPtr(), key.c_str(), key.length()); } else - { item = attachment->att_dsql_cache.get(key); - } return item; } diff --git a/src/jrd/met_proto.h b/src/jrd/met_proto.h index 4981bc7e3e..38716837c3 100644 --- a/src/jrd/met_proto.h +++ b/src/jrd/met_proto.h @@ -79,8 +79,8 @@ Jrd::DeferredWork* MET_change_fields(Jrd::thread_db*, Jrd::jrd_tra*, const dsc*) Jrd::Format* MET_current(Jrd::thread_db*, Jrd::jrd_rel*); void MET_delete_dependencies(Jrd::thread_db*, const Firebird::MetaName&, int, Jrd::jrd_tra*); void MET_delete_shadow(Jrd::thread_db*, USHORT); -bool MET_dsql_cache_use(Jrd::thread_db* tdbb, int type, const Firebird::MetaName& name, const Firebird::MetaName& package = ""); -void MET_dsql_cache_release(Jrd::thread_db* tdbb, int type, const Firebird::MetaName& name, const Firebird::MetaName& package = ""); +bool MET_dsql_cache_use(Jrd::thread_db* tdbb, Jrd::sym_type type, const Firebird::MetaName& name, const Firebird::MetaName& package = ""); +void MET_dsql_cache_release(Jrd::thread_db* tdbb, Jrd::sym_type type, const Firebird::MetaName& name, const Firebird::MetaName& package = ""); void MET_error(const TEXT*, ...); Jrd::Format* MET_format(Jrd::thread_db*, Jrd::jrd_rel*, USHORT); bool MET_get_char_coll_subtype(Jrd::thread_db*, USHORT*, const UCHAR*, USHORT); From 0ffbfddce7d068ef0b36e9d72dd3d522060536b0 Mon Sep 17 00:00:00 2001 From: stbergmann Date: Mon, 21 Oct 2019 19:48:08 +0200 Subject: [PATCH 082/274] Make comparison operator member functions const (#227) ...which avoids overload resolution ambiguities in C++20, when a synthesized candidate of operator == for a reversed-argument rewrite conflicts with the actual operator ==, due to the asymmetric const-ness of the implicit object parameter and the RHS parameter. (As observed with recent Clang 10 trunk with -std=c++2a when building firebird as part of LibreOffice: > workdir/UnpackedTarball/firebird/src/jrd/inf.cpp:1139:62: error: use of overloaded operator '!=' is ambiguous (with operand types 'RuntimeStatistics::Iterator' and 'Jrd::RuntimeStatistics::Iterator') > for (RuntimeStatistics::Iterator iter = stats.begin(); iter != stats.end(); ++iter) > ~~~~ ^ ~~~~~~~~~~~ > workdir/UnpackedTarball/firebird/src/jrd/../dsql/../jrd/RuntimeStatistics.h:283:8: note: candidate function > bool operator!=(const Iterator& other) > ^ > workdir/UnpackedTarball/firebird/src/jrd/../dsql/../jrd/RuntimeStatistics.h:278:8: note: candidate function > bool operator==(const Iterator& other) > ^ > workdir/UnpackedTarball/firebird/src/jrd/../dsql/../jrd/RuntimeStatistics.h:278:8: note: candidate function (with reversed parameter order) ) --- src/jrd/RuntimeStatistics.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jrd/RuntimeStatistics.h b/src/jrd/RuntimeStatistics.h index 74a03de2ad..fab286ad1a 100644 --- a/src/jrd/RuntimeStatistics.h +++ b/src/jrd/RuntimeStatistics.h @@ -290,12 +290,12 @@ public: {} public: - bool operator==(const Iterator& other) + bool operator==(const Iterator& other) const { return (m_counts == other.m_counts); } - bool operator!=(const Iterator& other) + bool operator!=(const Iterator& other) const { return (m_counts != other.m_counts); } From 5bec61dd164d605ef1d4a293c4fb430c1db2c06b Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Tue, 22 Oct 2019 00:03:55 +0000 Subject: [PATCH 083/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 88f804d728..4147558d3c 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1633 + FORMAL BUILD NUMBER:1635 */ -#define PRODUCT_VER_STRING "4.0.0.1633" -#define FILE_VER_STRING "WI-T4.0.0.1633" -#define LICENSE_VER_STRING "WI-T4.0.0.1633" -#define FILE_VER_NUMBER 4, 0, 0, 1633 +#define PRODUCT_VER_STRING "4.0.0.1635" +#define FILE_VER_STRING "WI-T4.0.0.1635" +#define LICENSE_VER_STRING "WI-T4.0.0.1635" +#define FILE_VER_NUMBER 4, 0, 0, 1635 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1633" +#define FB_BUILD_NO "1635" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index d6bf17b0d9..4732358287 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1633 +BuildNum=1635 NowAt=`pwd` cd `dirname $0` From bf51db842ec7e3234d569ccf163d92c3f1556b78 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 25 Oct 2019 13:22:28 +0300 Subject: [PATCH 084/274] Comment --- builds/install/misc/firebird.conf.in | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/builds/install/misc/firebird.conf.in b/builds/install/misc/firebird.conf.in index a417111651..eba66698db 100644 --- a/builds/install/misc/firebird.conf.in +++ b/builds/install/misc/firebird.conf.in @@ -176,15 +176,15 @@ # are stored. Relative paths are treated relative to the root directory # of firebird. # -# Default value 'Restrict UDF' provides the same restrictions -# as in FB 1.0. To specify access to specific trees, enum all required -# paths (for Windows this may be something like 'C:\ExternalFunctions', -# for unix - '/db/extern;/mnt/extern'). +# Since FB4.0 default value is None. Set it to 'Restrict UDF' to have +# the same restrictions as in previous FB versions. To specify access +# to specific trees, enum all required paths (for Windows this may be +# something like 'C:\ExternalFunctions', for unix - '/db/udf;/mnt/udf'). @UDF_COMMENT@ # NOTE: THE EXTERNAL FUNCTION ENGINE FEATURE COULD BE USED TO COMPROMISE # THE SERVER/HOST AS WELL AS DATABASE SECURITY!! # -# IT IS STRONGLY RECOMMENDED THAT THIS SETTING BE SET TO NONE! +# IT IS STRONGLY RECOMMENDED THAT THIS SETTING REMAINS NONE! # # Type: string (special format) # From 0453a7f90f8accd6f3ed5f980997330b417786f2 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 26 Oct 2019 00:03:52 +0000 Subject: [PATCH 085/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 4147558d3c..9457ddee4b 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1635 + FORMAL BUILD NUMBER:1636 */ -#define PRODUCT_VER_STRING "4.0.0.1635" -#define FILE_VER_STRING "WI-T4.0.0.1635" -#define LICENSE_VER_STRING "WI-T4.0.0.1635" -#define FILE_VER_NUMBER 4, 0, 0, 1635 +#define PRODUCT_VER_STRING "4.0.0.1636" +#define FILE_VER_STRING "WI-T4.0.0.1636" +#define LICENSE_VER_STRING "WI-T4.0.0.1636" +#define FILE_VER_NUMBER 4, 0, 0, 1636 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1635" +#define FB_BUILD_NO "1636" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 4732358287..440a10c109 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1635 +BuildNum=1636 NowAt=`pwd` cd `dirname $0` From 35e479b9b0ed1015e137072f38c6a8a0f593de1e Mon Sep 17 00:00:00 2001 From: Dimitry Sibiryakov Date: Mon, 28 Oct 2019 18:50:10 +0100 Subject: [PATCH 086/274] Fix Windows build in path with spaces. Update ignore lists (#228) --- builds/win32/compile.bat | 2 +- extern/icu/.gitignore | 4 ++-- extern/re2/.gitignore | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/builds/win32/compile.bat b/builds/win32/compile.bat index 5276cd7837..07ae6415e0 100644 --- a/builds/win32/compile.bat +++ b/builds/win32/compile.bat @@ -32,7 +32,7 @@ goto loop_start :loop_end -if not exist %output_path% mkdir %output_path% +if not exist "%output_path%" mkdir "%output_path%" msbuild "%FB_LONG_ROOT_PATH%\%solution%.sln" /maxcpucount /p:Configuration=%config% /p:Platform=%FB_TARGET_PLATFORM% %projects% /fileLoggerParameters:LogFile=%output% endlocal diff --git a/extern/icu/.gitignore b/extern/icu/.gitignore index 2961411d8b..fe570806bb 100644 --- a/extern/icu/.gitignore +++ b/extern/icu/.gitignore @@ -3,6 +3,6 @@ include/ lib/ Win32/ x64/ -icudt52l.dat -icudt52l_empty.dat +icudt63l.dat +icudt63l_empty.dat license.html diff --git a/extern/re2/.gitignore b/extern/re2/.gitignore index a671fe2cf0..ec01f39b64 100644 --- a/extern/re2/.gitignore +++ b/extern/re2/.gitignore @@ -3,3 +3,4 @@ core obj/ benchlog.* +builds/ From 297824a7b8601a179ac184695ac36b627030647c Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Tue, 29 Oct 2019 00:03:46 +0000 Subject: [PATCH 087/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 9457ddee4b..096d93d970 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1636 + FORMAL BUILD NUMBER:1637 */ -#define PRODUCT_VER_STRING "4.0.0.1636" -#define FILE_VER_STRING "WI-T4.0.0.1636" -#define LICENSE_VER_STRING "WI-T4.0.0.1636" -#define FILE_VER_NUMBER 4, 0, 0, 1636 +#define PRODUCT_VER_STRING "4.0.0.1637" +#define FILE_VER_STRING "WI-T4.0.0.1637" +#define LICENSE_VER_STRING "WI-T4.0.0.1637" +#define FILE_VER_NUMBER 4, 0, 0, 1637 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1636" +#define FB_BUILD_NO "1637" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 440a10c109..1623f6f191 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1636 +BuildNum=1637 NowAt=`pwd` cd `dirname $0` From 1ccadfde9c96a5c3f9eca6d1adb2391c6bb24a52 Mon Sep 17 00:00:00 2001 From: Dmitry Yemanov Date: Tue, 29 Oct 2019 15:06:19 +0300 Subject: [PATCH 088/274] Fixed CORE-6171: No current record for fetch operation with queries with aggregated subselect --- src/jrd/Optimizer.cpp | 4 ++-- src/jrd/opt.cpp | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/jrd/Optimizer.cpp b/src/jrd/Optimizer.cpp index 0fba4220cf..4389356695 100644 --- a/src/jrd/Optimizer.cpp +++ b/src/jrd/Optimizer.cpp @@ -2261,7 +2261,7 @@ InversionCandidate* OptimizerRetrieval::matchOnIndexes( { BoolExprNode* condition = binaryNode->arg2; - if (condition->computable(csb, INVALID_STREAM, false) && !condition->findStream(csb, stream)) + if (condition->computable(csb, stream, false)) { if (invCandidate1->condition) { @@ -2282,7 +2282,7 @@ InversionCandidate* OptimizerRetrieval::matchOnIndexes( { BoolExprNode* condition = binaryNode->arg1; - if (condition->computable(csb, INVALID_STREAM, false) && !condition->findStream(csb, stream)) + if (condition->computable(csb, stream, false)) { if (invCandidate2->condition) { diff --git a/src/jrd/opt.cpp b/src/jrd/opt.cpp index 7c5866bbbf..6bfbc70f93 100644 --- a/src/jrd/opt.cpp +++ b/src/jrd/opt.cpp @@ -2313,8 +2313,7 @@ static RecordSource* gen_retrieval(thread_db* tdbb, // inside OptimizerRetrieval::matchOnIndexes() if (inversion && condition && - (!condition->computable(csb, INVALID_STREAM, false) || - condition->findStream(csb, stream))) + !condition->computable(csb, stream, false)) { fb_assert(false); inversion = NULL; From d5daff04add11298cff468afc52433465306fe1e Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 30 Oct 2019 00:04:15 +0000 Subject: [PATCH 089/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 096d93d970..3390ed738f 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1637 + FORMAL BUILD NUMBER:1638 */ -#define PRODUCT_VER_STRING "4.0.0.1637" -#define FILE_VER_STRING "WI-T4.0.0.1637" -#define LICENSE_VER_STRING "WI-T4.0.0.1637" -#define FILE_VER_NUMBER 4, 0, 0, 1637 +#define PRODUCT_VER_STRING "4.0.0.1638" +#define FILE_VER_STRING "WI-T4.0.0.1638" +#define LICENSE_VER_STRING "WI-T4.0.0.1638" +#define FILE_VER_NUMBER 4, 0, 0, 1638 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1637" +#define FB_BUILD_NO "1638" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 1623f6f191..2e7cb50d00 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1637 +BuildNum=1638 NowAt=`pwd` cd `dirname $0` From 8a8d7787d0061da64b7a45e5735c68df47f29531 Mon Sep 17 00:00:00 2001 From: Dimitry Sibiryakov Date: Thu, 31 Oct 2019 10:59:37 +0100 Subject: [PATCH 090/274] CORE-5538 implementation (#229) * CORE-5538 implementation --- doc/README.gbak | 19 ++++++++ src/burp/burp.cpp | 66 ++++++++++++++++++++++++-- src/burp/burp.h | 2 + src/burp/burpswi.h | 4 ++ src/common/classes/ClumpletReader.cpp | 1 + src/include/firebird/impl/consts_pub.h | 2 + src/msgs/facilities2.sql | 2 +- src/msgs/messages2.sql | 3 ++ src/utilities/fbsvcmgr/fbsvcmgr.cpp | 2 + 9 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 doc/README.gbak diff --git a/doc/README.gbak b/doc/README.gbak new file mode 100644 index 0000000000..678880d12d --- /dev/null +++ b/doc/README.gbak @@ -0,0 +1,19 @@ +In Firebird 4.0 a new switch was added to gbak: -INCLUDE(_DATA). + +It takes one parameter which is "similar like" pattern matching +table names in a case-insensitive way. + +This switch, if provided, limit tables for which data is stored +or restored in/from backup file. + +Interaction between -INCLUDE_DATA and -SKIP_DATA switches for +a table is following: ++--------------------------------------------------+ +| | INCLUDE_DATA | +| |--------------------------------------| +| SKIP_DATA | NOT SET | MATCH | NOT MATCH | ++-----------+------------+------------+------------+ +| NOT SET | included | included | excluded | +| MATCH | excluded | excluded | excluded | +| NOT MATCH | excluded | included | excluded | ++-----------+------------+------------+------------+ diff --git a/src/burp/burp.cpp b/src/burp/burp.cpp index 2b863a9b9e..c8bc046f6e 100644 --- a/src/burp/burp.cpp +++ b/src/burp/burp.cpp @@ -703,6 +703,14 @@ int gbak(Firebird::UtilSvc* uSvc) } tdgbl->setupSkipData(argv[itr]); break; + case IN_SW_BURP_INCLUDE_DATA: + if (++itr >= argc) + { + BURP_error(389, true); + // missing regular expression to include tables + } + tdgbl->setupIncludeData(argv[itr]); + break; case IN_SW_BURP_ROLE: if (++itr >= argc) { @@ -2540,6 +2548,38 @@ void BurpGlobals::setupSkipData(const Firebird::string& regexp) } } +void BurpGlobals::setupIncludeData(const Firebird::string& regexp) +{ + if (includeDataMatcher) + { + BURP_error(390, true); + // msg 390 regular expression to include tables was already set + } + + // Compile include relation expressions + try + { + if (regexp.hasData()) + { + Firebird::string filter(regexp); + if (!uSvc->utf8FileNames()) + ISC_systemToUtf8(filter); + + BurpGlobals* tdgbl = BurpGlobals::getSpecific(); + + includeDataMatcher.reset(FB_NEW_POOL(tdgbl->getPool()) Firebird::SimilarToRegex( + tdgbl->getPool(), Firebird::SimilarToFlag::CASE_INSENSITIVE, + filter.c_str(), filter.length(), + "\\", 1)); + } + } + catch (const Firebird::Exception&) + { + Firebird::fatal_exception::raiseFmt( + "error while compiling regular expression \"%s\"", regexp.c_str()); + } +} + Firebird::string BurpGlobals::toSystem(const Firebird::PathName& from) { Firebird::string to = from.ToString(); @@ -2548,15 +2588,35 @@ Firebird::string BurpGlobals::toSystem(const Firebird::PathName& from) return to; } +namespace // for local symbols +{ + enum Pattern { NOT_SET = 0, MATCH = 1, NOT_MATCH = 2 }; + + Pattern checkPattern(Firebird::AutoPtr& matcher, + const char* name) + { + if (!matcher) + return NOT_SET; + + return matcher->matches(name, strlen(name))? MATCH : NOT_MATCH; + } +} + bool BurpGlobals::skipRelation(const char* name) { if (gbl_sw_meta) return true; - if (!skipDataMatcher) - return false; + // Fine-grained table controlling cases when data must be skipped for a table + static const bool result[3][3] = { + // Include filter + // NS M NM S + { false, false, true}, // NS k + { true, true, true}, // M i + { false, false, true} // NM p + }; - return skipDataMatcher->matches(name, strlen(name)); + return result[checkPattern(skipDataMatcher, name)][checkPattern(includeDataMatcher, name)]; } void BurpGlobals::read_stats(SINT64* stats) diff --git a/src/burp/burp.h b/src/burp/burp.h index 53947dde1b..c4eb6e18b9 100644 --- a/src/burp/burp.h +++ b/src/burp/burp.h @@ -1139,6 +1139,7 @@ public: ThreadData::restoreSpecific(); } void setupSkipData(const Firebird::string& regexp); + void setupIncludeData(const Firebird::string& regexp); bool skipRelation(const char* name); char veryEnd; @@ -1155,6 +1156,7 @@ public: bool firstMap; // this is the first time we entered get_mapping() bool stdIoMode; // stdin or stdout is used as backup file Firebird::AutoPtr skipDataMatcher; + Firebird::AutoPtr includeDataMatcher; public: Firebird::string toSystem(const Firebird::PathName& from); diff --git a/src/burp/burpswi.h b/src/burp/burpswi.h index 2cfc6e7758..97ba37fac4 100644 --- a/src/burp/burpswi.h +++ b/src/burp/burpswi.h @@ -96,6 +96,8 @@ const int IN_SW_BURP_KEYHOLD = 49; // name of KeyHolder plugin const int IN_SW_BURP_KEYNAME = 50; // name of crypt key const int IN_SW_BURP_CRYPT = 51; // name of crypt plugin +const int IN_SW_BURP_INCLUDE_DATA = 52; // backup data from tables + /**************************************************************************/ static const char* const BURP_SW_MODE_RO = "READ_ONLY"; @@ -175,6 +177,8 @@ static const Switches::in_sw_tab_t reference_burp_in_sw_table[] = // msg 277: @1SE(RVICE) use services manager {IN_SW_BURP_SKIP_DATA, isc_spb_res_skip_data, "SKIP_DATA", 0, 0, 0, false, false, 355, 6, NULL, boGeneral}, // msg 355: @1SKIP_DATA skip data for table + {IN_SW_BURP_INCLUDE_DATA, isc_spb_res_include_data, "INCLUDE_DATA", 0, 0, 0, false, false, 388, 7, NULL, boGeneral}, + // msg 388: @1INCLUDE(_DATA) backup data of table(s) {IN_SW_BURP_STATS, isc_spb_bkp_stat, "STATISTICS", 0, 0, 0, false, false, 361, 2, NULL, boGeneral}, // msg 361: @1ST(ATISTICS) TDRW show statistics: {-1, 0, " ", 0, 0, 0, false, false, 362, 0, NULL, boGeneral}, diff --git a/src/common/classes/ClumpletReader.cpp b/src/common/classes/ClumpletReader.cpp index d20ed8c5ee..cc420d0ae1 100644 --- a/src/common/classes/ClumpletReader.cpp +++ b/src/common/classes/ClumpletReader.cpp @@ -335,6 +335,7 @@ ClumpletReader::ClumpletType ClumpletReader::getClumpletType(UCHAR tag) const case isc_spb_res_fix_fss_metadata: case isc_spb_bkp_stat: case isc_spb_bkp_skip_data: + case isc_spb_bkp_include_data: case isc_spb_bkp_keyholder: case isc_spb_bkp_keyname: case isc_spb_bkp_crypt: diff --git a/src/include/firebird/impl/consts_pub.h b/src/include/firebird/impl/consts_pub.h index 9f34fc29f6..78417c986d 100644 --- a/src/include/firebird/impl/consts_pub.h +++ b/src/include/firebird/impl/consts_pub.h @@ -406,6 +406,7 @@ #define isc_spb_bkp_keyholder 16 #define isc_spb_bkp_keyname 17 #define isc_spb_bkp_crypt 18 +#define isc_spb_bkp_include_data 19 #define isc_spb_bkp_ignore_checksums 0x01 #define isc_spb_bkp_ignore_limbo 0x02 #define isc_spb_bkp_metadata_only 0x04 @@ -513,6 +514,7 @@ *****************************************/ #define isc_spb_res_skip_data isc_spb_bkp_skip_data +#define isc_spb_res_include_data isc_spb_bkp_include_data #define isc_spb_res_buffers 9 #define isc_spb_res_page_size 10 #define isc_spb_res_length 11 diff --git a/src/msgs/facilities2.sql b/src/msgs/facilities2.sql index bcf03c80dd..e6de13afef 100644 --- a/src/msgs/facilities2.sql +++ b/src/msgs/facilities2.sql @@ -9,7 +9,7 @@ set bulk_insert INSERT INTO FACILITIES (LAST_CHANGE, FACILITY, FAC_CODE, MAX_NUM ('2018-06-22 11:46:00', 'DYN', 8, 309) ('1996-11-07 13:39:40', 'INSTALL', 10, 1) ('1996-11-07 13:38:41', 'TEST', 11, 4) -('2018-04-26 20:40:00', 'GBAK', 12, 388) +('2018-04-26 20:40:00', 'GBAK', 12, 391) ('2019-04-13 21:10:00', 'SQLERR', 13, 1047) ('1996-11-07 13:38:42', 'SQLWARN', 14, 613) ('2018-02-27 14:50:31', 'JRD_BUGCHK', 15, 308) diff --git a/src/msgs/messages2.sql b/src/msgs/messages2.sql index 5963594b59..dbeb5f4343 100644 --- a/src/msgs/messages2.sql +++ b/src/msgs/messages2.sql @@ -2545,6 +2545,9 @@ ERROR: Backup incomplete', NULL, NULL); (NULL, NULL, 'restore.epp', NULL, 12, 385, NULL, 'Invalid reply from getInfo() when waiting for DB encryption', NULL, NULL); (NULL, NULL, 'restore.epp', NULL, 12, 386, NULL, 'Problems with just created database encryption', NULL, NULL); (NULL, 'get_trigger', 'restore.epp', NULL, 12, 387, NULL, 'Skipped trigger @1 on system table @2', NULL, NULL); +(NULL, 'burp_usage', 'burp.c', NULL, 12, 388, NULL, ' @1INCLUDE(_DATA) backup data of table(s)', NULL, NULL); +(NULL, NULL, 'burp.cpp', NULL, 12, 389, NULL, 'missing regular expression to include tables', NULL, NULL); +(NULL, NULL, 'burp.cpp', NULL, 12, 390, NULL, 'regular expression to include tables was already set', NULL, NULL); -- SQLERR (NULL, NULL, NULL, NULL, 13, 1, NULL, 'Firebird error', NULL, NULL); (NULL, NULL, NULL, NULL, 13, 74, NULL, 'Rollback not performed', NULL, NULL); diff --git a/src/utilities/fbsvcmgr/fbsvcmgr.cpp b/src/utilities/fbsvcmgr/fbsvcmgr.cpp index cb1bd5286f..6f2abc3fc7 100644 --- a/src/utilities/fbsvcmgr/fbsvcmgr.cpp +++ b/src/utilities/fbsvcmgr/fbsvcmgr.cpp @@ -404,6 +404,7 @@ const SvcSwitches backupOptions[] = {"bkp_no_triggers", putOption, 0, isc_spb_bkp_no_triggers, 0}, {"verbint", putIntArgument, 0, isc_spb_verbint, 0}, {"bkp_skip_data", putStringArgument, 0, isc_spb_bkp_skip_data, 0}, + {"bkp_include_data", putStringArgument, 0, isc_spb_bkp_include_data, 0}, {"bkp_stat", putStringArgument, 0, isc_spb_bkp_stat, 0 }, {"bkp_keyholder", putStringArgument, 0, isc_spb_bkp_keyholder, 0 }, {"bkp_keyname", putStringArgument, 0, isc_spb_bkp_keyname, 0 }, @@ -433,6 +434,7 @@ const SvcSwitches restoreOptions[] = {"res_metadata_only", putOption, 0, isc_spb_res_metadata_only, 0}, {"verbint", putIntArgument, 0, isc_spb_verbint, 0}, {"res_skip_data", putStringArgument, 0, isc_spb_res_skip_data, 0}, + {"res_include_data", putStringArgument, 0, isc_spb_res_include_data, 0}, {"res_stat", putStringArgument, 0, isc_spb_res_stat, 0 }, {"res_keyholder", putStringArgument, 0, isc_spb_res_keyholder, 0 }, {"res_keyname", putStringArgument, 0, isc_spb_res_keyname, 0 }, From 6600de0f948a04e44cc904957693a36b75970832 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Fri, 1 Nov 2019 00:03:59 +0000 Subject: [PATCH 091/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 3390ed738f..7bddd1a5c6 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1638 + FORMAL BUILD NUMBER:1639 */ -#define PRODUCT_VER_STRING "4.0.0.1638" -#define FILE_VER_STRING "WI-T4.0.0.1638" -#define LICENSE_VER_STRING "WI-T4.0.0.1638" -#define FILE_VER_NUMBER 4, 0, 0, 1638 +#define PRODUCT_VER_STRING "4.0.0.1639" +#define FILE_VER_STRING "WI-T4.0.0.1639" +#define LICENSE_VER_STRING "WI-T4.0.0.1639" +#define FILE_VER_NUMBER 4, 0, 0, 1639 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1638" +#define FB_BUILD_NO "1639" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 2e7cb50d00..74f59cf0f4 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1638 +BuildNum=1639 NowAt=`pwd` cd `dirname $0` From 46102b60842d68b82fc2c240e26b1acb031b3d27 Mon Sep 17 00:00:00 2001 From: Dimitry Sibiryakov Date: Thu, 31 Oct 2019 15:34:08 +0100 Subject: [PATCH 092/274] Fix mistype in CORE-5538 documentation --- doc/README.gbak | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/README.gbak b/doc/README.gbak index 678880d12d..941389dd35 100644 --- a/doc/README.gbak +++ b/doc/README.gbak @@ -1,4 +1,4 @@ -In Firebird 4.0 a new switch was added to gbak: -INCLUDE(_DATA). +In Firebird 4.0 a new switch was added to gbak: -INCLUDE(_DATA). It takes one parameter which is "similar like" pattern matching table names in a case-insensitive way. @@ -15,5 +15,5 @@ a table is following: +-----------+------------+------------+------------+ | NOT SET | included | included | excluded | | MATCH | excluded | excluded | excluded | -| NOT MATCH | excluded | included | excluded | +| NOT MATCH | included | included | excluded | +-----------+------------+------------+------------+ From ee143e06c4ca193c4108ac1cc7d98f9ae4d557e6 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Fri, 1 Nov 2019 13:04:21 -0300 Subject: [PATCH 093/274] Hope it fixes CORE-6174: ibase.h missing from nightly (Windows) builds. --- builds/win32/make_all.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/builds/win32/make_all.bat b/builds/win32/make_all.bat index a5a24ecd05..b10caa714c 100644 --- a/builds/win32/make_all.bat +++ b/builds/win32/make_all.bat @@ -87,6 +87,7 @@ findstr /V "@UDF_COMMENT@" %FB_ROOT_PATH%\builds\install\misc\firebird.conf.in > :: Headers copy %FB_ROOT_PATH%\src\extlib\ib_util.h %FB_OUTPUT_DIR%\include > nul copy %FB_ROOT_PATH%\src\jrd\perf.h %FB_OUTPUT_DIR%\include >nul +copy %FB_ROOT_PATH%\src\include\ibase.h %FB_OUTPUT_DIR%\include > nul copy %FB_ROOT_PATH%\src\include\gen\iberror.h %FB_OUTPUT_DIR%\include > nul :: New API headers From 6633400974797fb4a0f43a43cbd613a686329684 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 2 Nov 2019 00:03:48 +0000 Subject: [PATCH 094/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 7bddd1a5c6..642097c4c5 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1639 + FORMAL BUILD NUMBER:1641 */ -#define PRODUCT_VER_STRING "4.0.0.1639" -#define FILE_VER_STRING "WI-T4.0.0.1639" -#define LICENSE_VER_STRING "WI-T4.0.0.1639" -#define FILE_VER_NUMBER 4, 0, 0, 1639 +#define PRODUCT_VER_STRING "4.0.0.1641" +#define FILE_VER_STRING "WI-T4.0.0.1641" +#define LICENSE_VER_STRING "WI-T4.0.0.1641" +#define FILE_VER_NUMBER 4, 0, 0, 1641 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1639" +#define FB_BUILD_NO "1641" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 74f59cf0f4..05bd9badc6 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1639 +BuildNum=1641 NowAt=`pwd` cd `dirname $0` From 1f1c82dde2f68f4e8d6a3d513c538fa7c4822889 Mon Sep 17 00:00:00 2001 From: hvlad Date: Tue, 5 Nov 2019 19:00:32 +0200 Subject: [PATCH 095/274] Fixed wrong datatype used --- src/jrd/Attachment.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jrd/Attachment.h b/src/jrd/Attachment.h index 46c3229b70..31ca54450d 100644 --- a/src/jrd/Attachment.h +++ b/src/jrd/Attachment.h @@ -178,7 +178,7 @@ public: CommitNumber getSnapshotForVersion(CommitNumber version_cn); private: - UInt32Bitmap m_snapshots; // List of active snapshots as of the moment of time + Firebird::SparseBitmap m_snapshots; // List of active snapshots as of the moment of time CommitNumber m_lastCommit; // CN_ACTIVE here means object is not populated ULONG m_releaseCount; // Release event counter when list was last updated ULONG m_slots_used; // Snapshot slots used when list was last updated From 0d9cb6658657cf9851292c2bfbe8ec7625207ab3 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 6 Nov 2019 00:04:00 +0000 Subject: [PATCH 096/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 642097c4c5..958f5b895b 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1641 + FORMAL BUILD NUMBER:1642 */ -#define PRODUCT_VER_STRING "4.0.0.1641" -#define FILE_VER_STRING "WI-T4.0.0.1641" -#define LICENSE_VER_STRING "WI-T4.0.0.1641" -#define FILE_VER_NUMBER 4, 0, 0, 1641 +#define PRODUCT_VER_STRING "4.0.0.1642" +#define FILE_VER_STRING "WI-T4.0.0.1642" +#define LICENSE_VER_STRING "WI-T4.0.0.1642" +#define FILE_VER_NUMBER 4, 0, 0, 1642 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1641" +#define FB_BUILD_NO "1642" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 05bd9badc6..c19fc7c5ec 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1641 +BuildNum=1642 NowAt=`pwd` cd `dirname $0` From 0d33f4e5b1375929fc470d58412b1c53db02d04b Mon Sep 17 00:00:00 2001 From: hvlad Date: Wed, 6 Nov 2019 15:37:41 +0200 Subject: [PATCH 097/274] Fixed bug CORE-6182 : ExtConnPoolLifeTime acts as countdown for activity in MOST RECENT database (of several) rather then separate for each of used databases --- src/yvalve/MasterImplementation.cpp | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/yvalve/MasterImplementation.cpp b/src/yvalve/MasterImplementation.cpp index bb97ca2f88..d836f37959 100644 --- a/src/yvalve/MasterImplementation.cpp +++ b/src/yvalve/MasterImplementation.cpp @@ -349,19 +349,15 @@ public: timerHolder.init(); TimerEntry* curTimer = getTimer(timer); - if (!curTimer) - { - TimerEntry newTimer; - - newTimer.timer = timer; - newTimer.fireTime = curTime() + microSeconds; - timerQueue->add(newTimer); - timer->addRef(); - } + if (curTimer) + timerQueue->remove(curTimer); else - { - curTimer->fireTime = curTime() + microSeconds; - } + timer->addRef(); + + TimerEntry newTimer; + newTimer.timer = timer; + newTimer.fireTime = curTime() + microSeconds; + timerQueue->add(newTimer); timerWakeup->release(); } From 9d27cb4b7e75022a785d033db1e8c13a396dd0a5 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Thu, 7 Nov 2019 00:03:54 +0000 Subject: [PATCH 098/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 958f5b895b..130c0611ff 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1642 + FORMAL BUILD NUMBER:1643 */ -#define PRODUCT_VER_STRING "4.0.0.1642" -#define FILE_VER_STRING "WI-T4.0.0.1642" -#define LICENSE_VER_STRING "WI-T4.0.0.1642" -#define FILE_VER_NUMBER 4, 0, 0, 1642 +#define PRODUCT_VER_STRING "4.0.0.1643" +#define FILE_VER_STRING "WI-T4.0.0.1643" +#define LICENSE_VER_STRING "WI-T4.0.0.1643" +#define FILE_VER_NUMBER 4, 0, 0, 1643 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1642" +#define FB_BUILD_NO "1643" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index c19fc7c5ec..aa7399ac39 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1642 +BuildNum=1643 NowAt=`pwd` cd `dirname $0` From fce84939b7c1b985daa0aefe617766bb1e92ab15 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Thu, 7 Nov 2019 12:56:21 -0300 Subject: [PATCH 099/274] Revert events example to usable version. --- examples/interfaces/08.events.cpp | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/examples/interfaces/08.events.cpp b/examples/interfaces/08.events.cpp index a829960765..a0c51c301d 100644 --- a/examples/interfaces/08.events.cpp +++ b/examples/interfaces/08.events.cpp @@ -5,7 +5,6 @@ * * Example for the following interfaces: * IEvents - returned by queEvents(), used to cancel events monitoring - * IEventBlock - creates and handles various blocks needed to que events * IEventCallback - it's callback is invoked when event happens * * The contents of this file are subject to the Mozilla Public License Version @@ -47,13 +46,11 @@ namespace attachment(aAttachment), counter(0), status(master->getStatus()), - eventBlock(NULL), events(NULL), first(true) { - const char* names[] = {name, NULL}; - eventBlock = master->getUtilInterface()->createEventBlock(&status, names); - events = attachment->queEvents(&status, this, eventBlock->getLength(), eventBlock->getValues()); + eveLen = isc_event_block(&eveBuffer, &eveResult, 1, name); + events = attachment->queEvents(&status, this, eveLen, eveBuffer); } void process(int pass) @@ -61,16 +58,15 @@ namespace if (!events) return; - unsigned tot = 0; + ISC_ULONG tot = 0; if (counter) { - eventBlock->counts(); - tot = eventBlock->getCounters()[0]; + isc_event_counts(&tot, eveLen, eveBuffer, eveResult); events->release(); events = NULL; counter = 0; - events = attachment->queEvents(&status, this, eventBlock->getLength(), eventBlock->getValues()); + events = attachment->queEvents(&status, this, eveLen, eveBuffer); } if (tot && !first) @@ -84,7 +80,7 @@ namespace // IEventCallback implementation void eventCallbackFunction(unsigned int length, const ISC_UCHAR* data) { - memcpy(eventBlock->getBuffer(), data, length); + memcpy(eveResult, data, length); ++counter; if (!first) printf("AST called\n"); @@ -112,7 +108,10 @@ namespace { if (events) events->release(); - eventBlock->dispose(); + if (eveBuffer) + isc_free((char*) eveBuffer); + if (eveResult) + isc_free((char*) eveResult); status.dispose(); } @@ -120,8 +119,10 @@ namespace IAttachment* attachment; volatile int counter; ThrowStatusWrapper status; - IEventBlock* eventBlock; IEvents* events; + unsigned char* eveBuffer; + unsigned char* eveResult; + unsigned eveLen; bool first; }; } From 98c77886497f2db48e566c8b3242cc86c2d364ed Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Fri, 8 Nov 2019 00:03:58 +0000 Subject: [PATCH 100/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 130c0611ff..238f4c2f43 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1643 + FORMAL BUILD NUMBER:1644 */ -#define PRODUCT_VER_STRING "4.0.0.1643" -#define FILE_VER_STRING "WI-T4.0.0.1643" -#define LICENSE_VER_STRING "WI-T4.0.0.1643" -#define FILE_VER_NUMBER 4, 0, 0, 1643 +#define PRODUCT_VER_STRING "4.0.0.1644" +#define FILE_VER_STRING "WI-T4.0.0.1644" +#define LICENSE_VER_STRING "WI-T4.0.0.1644" +#define FILE_VER_NUMBER 4, 0, 0, 1644 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1643" +#define FB_BUILD_NO "1644" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index aa7399ac39..f1f9226cdc 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1643 +BuildNum=1644 NowAt=`pwd` cd `dirname $0` From 1827c5617db07b3b74b095f41df5b3a380ac4788 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 8 Nov 2019 16:12:40 +0300 Subject: [PATCH 101/274] Fixed CORE-6181: Operations when using "SET DECFLOAT BIND BIGINT,n" with result of 11+ digits, fail with "Decimal float invalid operation" --- src/common/DecFloat.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/DecFloat.cpp b/src/common/DecFloat.cpp index 75d330119a..4d65d55282 100644 --- a/src/common/DecFloat.cpp +++ b/src/common/DecFloat.cpp @@ -123,12 +123,12 @@ private: } }; +const CDecimal128 pow2_32("4294967296", DecimalStatus(0)); +const CDecimal128 pow2_64("18446744073709551616", DecimalStatus(0)); const CDecimal128 dmax(DBL_MAX, DecimalStatus(0)), dmin(-DBL_MAX, DecimalStatus(0)); const CDecimal128 dzup(DBL_MIN, DecimalStatus(0)), dzlw(-DBL_MIN, DecimalStatus(0)); const CDecimal128 i64max(MAX_SINT64, DecimalStatus(0)), i64min(MIN_SINT64, DecimalStatus(0)); const CDecimal128 c1(1); -const CDecimal128 pow2_32("4294967296", DecimalStatus(0)); -const CDecimal128 pow2_64("18446744073709551616", DecimalStatus(0)); unsigned digits(const unsigned pMax, unsigned char* const coeff, int& exp) { From 76cacb9366d1fb313413e5345f99d913bd14270e Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 8 Nov 2019 16:31:28 +0300 Subject: [PATCH 102/274] Comment --- src/common/DecFloat.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/DecFloat.cpp b/src/common/DecFloat.cpp index 4d65d55282..eca87138b1 100644 --- a/src/common/DecFloat.cpp +++ b/src/common/DecFloat.cpp @@ -127,6 +127,7 @@ const CDecimal128 pow2_32("4294967296", DecimalStatus(0)); const CDecimal128 pow2_64("18446744073709551616", DecimalStatus(0)); const CDecimal128 dmax(DBL_MAX, DecimalStatus(0)), dmin(-DBL_MAX, DecimalStatus(0)); const CDecimal128 dzup(DBL_MIN, DecimalStatus(0)), dzlw(-DBL_MIN, DecimalStatus(0)); +// be careful with order of initialization: pow2_32 is used in i64max constructor const CDecimal128 i64max(MAX_SINT64, DecimalStatus(0)), i64min(MIN_SINT64, DecimalStatus(0)); const CDecimal128 c1(1); From bb0ed29e19547069874c4827d0422264c2822065 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 9 Nov 2019 00:03:34 +0000 Subject: [PATCH 103/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 238f4c2f43..45321d73c7 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1644 + FORMAL BUILD NUMBER:1646 */ -#define PRODUCT_VER_STRING "4.0.0.1644" -#define FILE_VER_STRING "WI-T4.0.0.1644" -#define LICENSE_VER_STRING "WI-T4.0.0.1644" -#define FILE_VER_NUMBER 4, 0, 0, 1644 +#define PRODUCT_VER_STRING "4.0.0.1646" +#define FILE_VER_STRING "WI-T4.0.0.1646" +#define LICENSE_VER_STRING "WI-T4.0.0.1646" +#define FILE_VER_NUMBER 4, 0, 0, 1646 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1644" +#define FB_BUILD_NO "1646" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index f1f9226cdc..8b9b06c867 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1644 +BuildNum=1646 NowAt=`pwd` cd `dirname $0` From dac8634de9a2eeb4fbc1087d4fbcdef4fc4f6821 Mon Sep 17 00:00:00 2001 From: Dimitry Sibiryakov Date: Tue, 12 Nov 2019 13:42:49 +0100 Subject: [PATCH 104/274] Fix warning on Win64 build (#231) --- src/common/ThreadStart.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/common/ThreadStart.cpp b/src/common/ThreadStart.cpp index c3d074c5b9..68e3d61872 100644 --- a/src/common/ThreadStart.cpp +++ b/src/common/ThreadStart.cpp @@ -309,13 +309,16 @@ ThreadId Thread::start(ThreadEntryPoint* routine, void* arg, int priority_arg, H * Advanced Windows by Richter pg. # 109. */ unsigned thread_id; - unsigned long real_handle = - _beginthreadex(NULL, 0, THREAD_ENTRYPOINT, THREAD_ARG, CREATE_SUSPENDED, &thread_id); - if (!real_handle) + HANDLE handle = + reinterpret_cast(_beginthreadex(NULL, 0, THREAD_ENTRYPOINT, THREAD_ARG, CREATE_SUSPENDED, &thread_id)); + if (!handle) { + // Though MSDN says that _beginthreadex() returns error in errno, + // GetLastError() still works because RTL call no other system + // functions after CreateThread() in the case of error. + // Watch out if it is ever changed. Firebird::system_call_failed::raise("_beginthreadex", GetLastError()); } - HANDLE handle = reinterpret_cast(real_handle); SetThreadPriority(handle, priority); From 5005bfc72fb05ddb75ad5759022a1ea78800bdb0 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 13 Nov 2019 00:03:52 +0000 Subject: [PATCH 105/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 45321d73c7..a606e4a0a8 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1646 + FORMAL BUILD NUMBER:1647 */ -#define PRODUCT_VER_STRING "4.0.0.1646" -#define FILE_VER_STRING "WI-T4.0.0.1646" -#define LICENSE_VER_STRING "WI-T4.0.0.1646" -#define FILE_VER_NUMBER 4, 0, 0, 1646 +#define PRODUCT_VER_STRING "4.0.0.1647" +#define FILE_VER_STRING "WI-T4.0.0.1647" +#define LICENSE_VER_STRING "WI-T4.0.0.1647" +#define FILE_VER_NUMBER 4, 0, 0, 1647 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1646" +#define FB_BUILD_NO "1647" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 8b9b06c867..53e5d31d4c 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1646 +BuildNum=1647 NowAt=`pwd` cd `dirname $0` From c8a8ad616be34d197d2fe9f370bf9e3355cc03a2 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Wed, 13 Nov 2019 11:43:33 -0300 Subject: [PATCH 106/274] Fixed CORE-5902 - Add Firebird Event fails with error (#232) * Fixed CORE-5902 - Add Firebird Event fails with error "While isc_que_events - Failed to establish a secondary connection for event processing". --- src/remote/SockAddr.h | 33 +++++++++++++++++++++++++++++++++ src/remote/inet.cpp | 26 +++++++++++++++++++++++--- src/remote/remote.h | 3 ++- src/remote/server/server.cpp | 1 + 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/remote/SockAddr.h b/src/remote/SockAddr.h index c1ab913c7e..624c9df5de 100644 --- a/src/remote/SockAddr.h +++ b/src/remote/SockAddr.h @@ -48,10 +48,24 @@ class SockAddr { private: + struct SockAddrPrefixPosixWindows + { + uint16_t sa_family; + }; + + struct SockAddrPrefixMacOs + { + uint8_t sa_len; + uint8_t sa_family; + }; + union sa_data { struct sockaddr sock; struct sockaddr_in inet; struct sockaddr_in6 inet6; + + SockAddrPrefixPosixWindows posixWindowsPrefix; + SockAddrPrefixMacOs macOsPrefix; } data; socklen_t len; static const unsigned MAX_LEN = sizeof(sa_data); @@ -59,6 +73,8 @@ private: void checkAndFixFamily(); public: + void convertFromMacOsToPosixWindows(); + void convertFromPosixWindowsToMacOs(); void clear(); const SockAddr& operator = (const SockAddr& x); @@ -120,6 +136,23 @@ inline void SockAddr::checkAndFixFamily() } } +inline void SockAddr::convertFromMacOsToPosixWindows() +{ + SockAddrPrefixMacOs macOsPrefix; + macOsPrefix = data.macOsPrefix; + + data.posixWindowsPrefix.sa_family = macOsPrefix.sa_family; +} + +inline void SockAddr::convertFromPosixWindowsToMacOs() +{ + SockAddrPrefixPosixWindows posixWindowsPrefix; + posixWindowsPrefix = data.posixWindowsPrefix; + + data.macOsPrefix.sa_family = posixWindowsPrefix.sa_family; + data.macOsPrefix.sa_len = length(); +} + inline void SockAddr::clear() { diff --git a/src/remote/inet.cpp b/src/remote/inet.cpp index 87e8235c2d..689ccf014f 100644 --- a/src/remote/inet.cpp +++ b/src/remote/inet.cpp @@ -1480,6 +1480,7 @@ static rem_port* aux_connect(rem_port* port, PACKET* packet) port->auxAcceptError(packet); inet_error(false, port, "socket", isc_net_event_connect_err, savedError); } + SockAddr resp_address(response->p_resp_data.cstr_address, response->p_resp_data.cstr_length); address.setPort(resp_address.port()); @@ -1577,7 +1578,6 @@ static rem_port* aux_request( rem_port* port, PACKET* packet) inet_error(false, port, "listen", isc_net_event_listen_err, INET_ERRNO); } - rem_port* const new_port = alloc_port(port->port_parent, (port->port_flags & PORT_no_oob) | PORT_async | PORT_connecting); port->port_async = new_port; @@ -1592,12 +1592,32 @@ static rem_port* aux_request( rem_port* port, PACKET* packet) P_RESP* response = &packet->p_resp; SockAddr port_address; + if (port_address.getsockname(port->port_handle) < 0) - { inet_error(false, port, "getsockname", isc_net_event_listen_err, INET_ERRNO); - } + port_address.setPort(our_address.port()); + // CORE-5902: MacOS has sockaddr struct layout different than one found in POSIX/Windows. + // This prevent usage of events when client or server is MacOS but the other end is not. + // Here we try to make this case work. However it's not bullet-proof for others platforms and architectures. + // A proper solution would be to just send the port number in a protocol friendly way. + + bool macOsClient = + port->port_client_arch == arch_darwin_ppc || + port->port_client_arch == arch_darwin_x64 || + port->port_client_arch == arch_darwin_ppc64; + + bool macOsServer = + ARCHITECTURE == arch_darwin_ppc || + ARCHITECTURE == arch_darwin_x64 || + ARCHITECTURE == arch_darwin_ppc64; + + if (macOsServer && !macOsClient) + port_address.convertFromMacOsToPosixWindows(); + else if (!macOsServer && macOsClient) + port_address.convertFromPosixWindowsToMacOs(); + response->p_resp_data.cstr_length = (ULONG) port_address.length(); memcpy(response->p_resp_data.cstr_address, port_address.ptr(), port_address.length()); diff --git a/src/remote/remote.h b/src/remote/remote.h index 8d0801b5fd..a8052368bd 100644 --- a/src/remote/remote.h +++ b/src/remote/remote.h @@ -1014,6 +1014,7 @@ struct rem_port : public Firebird::GlobalStorage, public Firebird::RefCounted rem_str* port_version; rem_str* port_host; // Our name rem_str* port_connection; // Name of connection + P_ARCH port_client_arch; Firebird::string port_login; Firebird::string port_user_name; Firebird::string port_peer_name; @@ -1078,7 +1079,7 @@ public: port_packet_vector(0), #endif port_objects(getPool()), port_version(0), port_host(0), - port_connection(0), port_login(getPool()), + port_connection(0), port_client_arch(arch_generic), port_login(getPool()), port_user_name(getPool()), port_peer_name(getPool()), port_protocol_id(getPool()), port_address(getPool()), port_rpr(0), port_statement(0), port_receive_rmtque(0), diff --git a/src/remote/server/server.cpp b/src/remote/server/server.cpp index c6e6772390..354395e777 100644 --- a/src/remote/server/server.cpp +++ b/src/remote/server/server.cpp @@ -1884,6 +1884,7 @@ static bool accept_connection(rem_port* port, P_CNCT* connect, PACKET* send) if (type == ptype_lazy_send) port->port_flags |= PORT_lazy; + port->port_client_arch = connect->p_cnct_client; Firebird::ClumpletReader id(Firebird::ClumpletReader::UnTagged, connect->p_cnct_user_id.cstr_address, From 6260af2a781c3ed6d2806ad1738e446b25098232 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Thu, 14 Nov 2019 00:05:36 +0000 Subject: [PATCH 107/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index a606e4a0a8..0db99c9c33 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1647 + FORMAL BUILD NUMBER:1648 */ -#define PRODUCT_VER_STRING "4.0.0.1647" -#define FILE_VER_STRING "WI-T4.0.0.1647" -#define LICENSE_VER_STRING "WI-T4.0.0.1647" -#define FILE_VER_NUMBER 4, 0, 0, 1647 +#define PRODUCT_VER_STRING "4.0.0.1648" +#define FILE_VER_STRING "WI-T4.0.0.1648" +#define LICENSE_VER_STRING "WI-T4.0.0.1648" +#define FILE_VER_NUMBER 4, 0, 0, 1648 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1647" +#define FB_BUILD_NO "1648" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 53e5d31d4c..6d8e0ecfea 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1647 +BuildNum=1648 NowAt=`pwd` cd `dirname $0` From 94d3a7cfe8fd50b45e82112b1c477330f388df2f Mon Sep 17 00:00:00 2001 From: hvlad Date: Fri, 15 Nov 2019 10:54:50 +0200 Subject: [PATCH 108/274] Let external transaction run with CONCURRENCY isolation mode if local transaction runs with READ COMMITED READ CONSISTENCY isolation and such isolation mode is not supported by external data source. Allow to reuse external connection if external data source doesn't support ALTER SESSION RESET statement. Update documentation. --- doc/sql.extensions/README.execute_statement2 | 6 +- .../README.external_connections_pool | 4 +- src/jrd/extds/ExtDS.cpp | 3 +- src/jrd/extds/ExtDS.h | 17 ++++++ src/jrd/extds/InternalDS.cpp | 2 + src/jrd/extds/IscDS.cpp | 55 +++++++++++++++++-- src/jrd/extds/IscDS.h | 3 + 7 files changed, 83 insertions(+), 7 deletions(-) diff --git a/doc/sql.extensions/README.execute_statement2 b/doc/sql.extensions/README.execute_statement2 index 072012d52b..3476fd69da 100644 --- a/doc/sql.extensions/README.execute_statement2 +++ b/doc/sql.extensions/README.execute_statement2 @@ -76,7 +76,11 @@ Syntax and notes : This transaction lifetime is bound to the lifetime of current (local) transaction and commits\rolled back the same way as current transaction. -- by default COMMON TRANSACTION is used +- by default COMMON TRANSACTION is used. + +- if local transaction is READ COMMITTED READ CONSISTENCY and if external data source not + supported this isolation mode, then external transaction will run with CONCURRENCY + isolation mode. - if PASSWORD clause is omitted then a) if is omitted, NULL or equal to CURRENT_USER value and diff --git a/doc/sql.extensions/README.external_connections_pool b/doc/sql.extensions/README.external_connections_pool index 3262bfcded..389352c97f 100644 --- a/doc/sql.extensions/README.external_connections_pool +++ b/doc/sql.extensions/README.external_connections_pool @@ -15,7 +15,9 @@ How pool works: active transactions), it is reset and placed into idle list (on successful reset) or closed (if reset failed). Connection is reset using ALTER SESSION RESET statement. It is considered - successful if no error occurs. + successful if no error occurs. Note, if external data source not supported + ALTER SESSION RESET statement - it is not considered as error and such + connection will be placed into pool. - if the pool has reached max. size, the oldest idle connection is closed - when engine ask to create a new external connection, the pool first looks for candidate at the idle list. The search is based on 4 parameters: diff --git a/src/jrd/extds/ExtDS.cpp b/src/jrd/extds/ExtDS.cpp index b3bba18b79..aae9ef3589 100644 --- a/src/jrd/extds/ExtDS.cpp +++ b/src/jrd/extds/ExtDS.cpp @@ -525,7 +525,8 @@ Connection::Connection(Provider& prov) : m_deleting(false), m_sqlDialect(0), m_wrapErrors(true), - m_broken(false) + m_broken(false), + m_features(0) { } diff --git a/src/jrd/extds/ExtDS.h b/src/jrd/extds/ExtDS.h index 247eff218d..d6838e35eb 100644 --- a/src/jrd/extds/ExtDS.h +++ b/src/jrd/extds/ExtDS.h @@ -437,6 +437,8 @@ public: virtual void detach(Jrd::thread_db* tdbb); virtual bool cancelExecution(bool forced) = 0; + + // Try to reset connection, return true if it can be pooled virtual bool resetSession() = 0; int getSqlDialect() const { return m_sqlDialect; } @@ -492,6 +494,13 @@ public: virtual Blob* createBlob() = 0; + // Test specified flags, return true if all bits present + bool testFeature(ULONG value) const { return m_features & value; } + // Set specified flags, return new value + ULONG setFeature(ULONG value) { return m_features |= value; } + // Clear specified flags, return new value + ULONG clearFeature(ULONG value) { return m_features &= ~value; } + protected: virtual Transaction* doCreateTransaction() = 0; virtual Statement* doCreateStatement() = 0; @@ -522,8 +531,16 @@ protected: int m_sqlDialect; // must be filled in attach call bool m_wrapErrors; bool m_broken; + ULONG m_features; // bitmask }; +// Connection features flags +const ULONG conFtrSessionReset = 0x01; // supports ALTER SESSION RESET +const ULONG conFtrReadConsistency = 0x02; // supports READ COMMITTED READ CONSISTENCY +const ULONG conFtrStatementTimeout = 0x04; // supports statements timeout + +// Features of Firebird 4 +const ULONG conFtrFB4 = conFtrSessionReset | conFtrReadConsistency | conFtrStatementTimeout; class Transaction : public Firebird::PermanentStorage { diff --git a/src/jrd/extds/InternalDS.cpp b/src/jrd/extds/InternalDS.cpp index 9057edf4e4..0b510ea8de 100644 --- a/src/jrd/extds/InternalDS.cpp +++ b/src/jrd/extds/InternalDS.cpp @@ -187,6 +187,8 @@ void InternalConnection::attach(thread_db* tdbb) m_sqlDialect = (attachment->att_database->dbb_flags & DBB_DB_SQL_dialect_3) ? SQL_DIALECT_V6 : SQL_DIALECT_V5; + + m_features = conFtrFB4; } void InternalConnection::doDetach(thread_db* tdbb) diff --git a/src/jrd/extds/IscDS.cpp b/src/jrd/extds/IscDS.cpp index 43ab8a5286..9d4847b938 100644 --- a/src/jrd/extds/IscDS.cpp +++ b/src/jrd/extds/IscDS.cpp @@ -191,6 +191,8 @@ void IscConnection::attach(thread_db* tdbb) } p += len; } + + m_features = conFtrFB4; // Exact feature set will be detected at first usage } void IscConnection::doDetach(thread_db* tdbb) @@ -238,6 +240,9 @@ bool IscConnection::resetSession() if (!m_handle) return false; + if (!testFeature(conFtrSessionReset)) + return true; + FbLocalStatus status; m_iscProvider.isc_dsql_execute_immediate(&status, &m_handle, NULL, 0, "ALTER SESSION RESET", m_sqlDialect, NULL); @@ -245,7 +250,13 @@ bool IscConnection::resetSession() if (!(status->getState() & IStatus::STATE_ERRORS)) return true; - return false; // (status->getErrors()[1] == isc_dsql_error); + if (status->getErrors()[1] == isc_dsql_error) + { + clearFeature(conFtrSessionReset); + return true; + } + + return false; } // this ISC connection instance is available for the current execution context if it @@ -301,14 +312,44 @@ Statement* IscConnection::doCreateStatement() // IscTransaction +void IscTransaction::generateTPB(thread_db* tdbb, ClumpletWriter& tpb, + TraModes traMode, bool readOnly, bool wait, int lockTimeout) const +{ + if (traMode == traReadCommitedReadConsistency && !m_connection.testFeature(conFtrReadConsistency)) + traMode = traConcurrency; + + Transaction::generateTPB(tdbb, tpb, traMode, readOnly, wait, lockTimeout); +} + void IscTransaction::doStart(FbStatusVector* status, thread_db* tdbb, Firebird::ClumpletWriter& tpb) { fb_assert(!m_handle); FB_API_HANDLE& db_handle = m_iscConnection.getAPIHandle(); - EngineCallbackGuard guard(tdbb, *this, FB_FUNCTION); - m_iscProvider.isc_start_transaction(status, &m_handle, 1, &db_handle, - tpb.getBufferLength(), tpb.getBuffer()); + { + EngineCallbackGuard guard(tdbb, *this, FB_FUNCTION); + m_iscProvider.isc_start_transaction(status, &m_handle, 1, &db_handle, + tpb.getBufferLength(), tpb.getBuffer()); + } + + if ((status->getState() & IStatus::STATE_ERRORS) && + (status->getErrors()[1] == isc_bad_tpb_form) && + tpb.find(isc_tpb_read_consistency) && + m_connection.testFeature(conFtrReadConsistency)) + { + tpb.deleteWithTag(isc_tpb_read_committed); + tpb.deleteWithTag(isc_tpb_read_consistency); + tpb.insertTag(isc_tpb_concurrency); + + { + EngineCallbackGuard guard(tdbb, *this, FB_FUNCTION); + m_iscProvider.isc_start_transaction(status, &m_handle, 1, &db_handle, + tpb.getBufferLength(), tpb.getBuffer()); + } + + if (!(status->getState() & IStatus::STATE_ERRORS)) + m_connection.clearFeature(conFtrReadConsistency); + } } void IscTransaction::doPrepare(FbStatusVector* /*status*/, thread_db* /*tdbb*/, int /*info_len*/, const char* /*info*/) @@ -521,6 +562,9 @@ void IscStatement::doPrepare(thread_db* tdbb, const string& sql) void IscStatement::doSetTimeout(thread_db* tdbb, unsigned int timeout) { + if (!m_connection.testFeature(conFtrStatementTimeout)) + return; + FbLocalStatus status; { @@ -533,7 +577,10 @@ void IscStatement::doSetTimeout(thread_db* tdbb, unsigned int timeout) // silently ignore error if timeouts is not supported by remote server // or loaded client library if (status[0] == isc_arg_gds && (status[1] == isc_wish_list || status[1] == isc_unavailable)) + { + m_connection.clearFeature(conFtrStatementTimeout); return; + } raise(&status, tdbb, "fb_dsql_set_timeout"); } diff --git a/src/jrd/extds/IscDS.h b/src/jrd/extds/IscDS.h index 8f8301ea13..6f2a6e5307 100644 --- a/src/jrd/extds/IscDS.h +++ b/src/jrd/extds/IscDS.h @@ -556,6 +556,9 @@ protected: virtual ~IscTransaction() {} + virtual void generateTPB(Jrd::thread_db* tdbb, Firebird::ClumpletWriter& tpb, + TraModes traMode, bool readOnly, bool wait, int lockTimeout) const; + virtual void doStart(Jrd::FbStatusVector* status, Jrd::thread_db* tdbb, Firebird::ClumpletWriter& tpb); virtual void doPrepare(Jrd::FbStatusVector* status, Jrd::thread_db* tdbb, int info_len, const char* info); virtual void doCommit(Jrd::FbStatusVector* status, Jrd::thread_db* tdbb, bool retain); From 460c7a96498cfb7bdbd1f2edeb07c89327bce15d Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 15 Nov 2019 14:44:01 +0300 Subject: [PATCH 109/274] Fixed example --- examples/udr/Functions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/udr/Functions.cpp b/examples/udr/Functions.cpp index ecbbc7d2dd..8c77e24156 100644 --- a/examples/udr/Functions.cpp +++ b/examples/udr/Functions.cpp @@ -156,7 +156,7 @@ FB_UDR_END_FUNCTION /*** create function mult ( a decfloat(34) not null, - b decimal(34,6) not null + b decfloat(34) not null ) returns decfloat(34) not null external name 'udrcpp_example!mult' engine udr; From 5e9000e442438ad0bb294314c89989c442985409 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 16 Nov 2019 00:03:54 +0000 Subject: [PATCH 110/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 0db99c9c33..62904e395d 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1648 + FORMAL BUILD NUMBER:1650 */ -#define PRODUCT_VER_STRING "4.0.0.1648" -#define FILE_VER_STRING "WI-T4.0.0.1648" -#define LICENSE_VER_STRING "WI-T4.0.0.1648" -#define FILE_VER_NUMBER 4, 0, 0, 1648 +#define PRODUCT_VER_STRING "4.0.0.1650" +#define FILE_VER_STRING "WI-T4.0.0.1650" +#define LICENSE_VER_STRING "WI-T4.0.0.1650" +#define FILE_VER_NUMBER 4, 0, 0, 1650 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1648" +#define FB_BUILD_NO "1650" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 6d8e0ecfea..2d49353cf7 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1648 +BuildNum=1650 NowAt=`pwd` cd `dirname $0` From a345149cfd8b190d9926445ae849b1d1d6c68222 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Mon, 18 Nov 2019 17:01:12 +0300 Subject: [PATCH 111/274] Comment --- src/jrd/ods.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/jrd/ods.h b/src/jrd/ods.h index 433944e9e9..9b3d432906 100644 --- a/src/jrd/ods.h +++ b/src/jrd/ods.h @@ -649,12 +649,14 @@ struct blh ULONG blh_length; // Total length of data USHORT blh_sub_type; // Blob sub-type UCHAR blh_charset; // Blob charset (since ODS 11.1) + UCHAR blh_unused; #ifndef ODS_TESTING +// Macro CHECK_BLOB_FIELD_ACCESS_FOR_SELECT is never defined, code under it was left for a case +// we would like to have that check in a future. #ifdef CHECK_BLOB_FIELD_ACCESS_FOR_SELECT USHORT blh_fld_id; // Field ID #endif #endif //ODS_TESTING - UCHAR blh_unused; ULONG blh_page[1]; // Page vector for blob pages }; From e7a728a80d6e1f3eb50ffc9dd7a06891e16eea09 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Tue, 19 Nov 2019 00:03:43 +0000 Subject: [PATCH 112/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 62904e395d..4602b6b5f1 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1650 + FORMAL BUILD NUMBER:1651 */ -#define PRODUCT_VER_STRING "4.0.0.1650" -#define FILE_VER_STRING "WI-T4.0.0.1650" -#define LICENSE_VER_STRING "WI-T4.0.0.1650" -#define FILE_VER_NUMBER 4, 0, 0, 1650 +#define PRODUCT_VER_STRING "4.0.0.1651" +#define FILE_VER_STRING "WI-T4.0.0.1651" +#define LICENSE_VER_STRING "WI-T4.0.0.1651" +#define FILE_VER_NUMBER 4, 0, 0, 1651 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1650" +#define FB_BUILD_NO "1651" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 2d49353cf7..7c27974401 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1650 +BuildNum=1651 NowAt=`pwd` cd `dirname $0` From 72d7f7615961e2a89eccf36acf6731dd1f6fd7d5 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sat, 23 Nov 2019 17:10:56 -0300 Subject: [PATCH 113/274] Fix memory leak in INET_connect() reported by Dimitry S. --- src/remote/inet.cpp | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/src/remote/inet.cpp b/src/remote/inet.cpp index 689ccf014f..a1bb0ce575 100644 --- a/src/remote/inet.cpp +++ b/src/remote/inet.cpp @@ -879,15 +879,25 @@ rem_port* INET_connect(const TEXT* name, #endif AI_ADDRCONFIG | (packet ? 0 : AI_PASSIVE); + struct AutoAddrInfo + { + ~AutoAddrInfo() + { + if (ptr) + freeaddrinfo(ptr); + } + + addrinfo* ptr = nullptr; + } gai_result; + const char* host_str = (host.hasData() ? host.c_str() : NULL); - struct addrinfo* gai_result; bool retry_gai; int n; do { retry_gai = false; - n = getaddrinfo(host_str, protocol.c_str(), &gai_hints, &gai_result); + n = getaddrinfo(host_str, protocol.c_str(), &gai_hints, &gai_result.ptr); if ((n == EAI_FAMILY || (!host_str && n == EAI_NONAME)) && (gai_hints.ai_family == AF_INET6) && (af != AF_INET6)) @@ -912,7 +922,7 @@ rem_port* INET_connect(const TEXT* name, inet_gen_error(true, port, Arg::Gds(isc_net_lookup_err) << Arg::Gds(isc_host_unknown)); } - for (const addrinfo* pai = gai_result; pai; pai = pai->ai_next) + for (const addrinfo* pai = gai_result.ptr; pai; pai = pai->ai_next) { // Allocate a port block and initialize a socket for communications port->port_handle = os_utils::socket(pai->ai_family, pai->ai_socktype, pai->ai_protocol); @@ -935,30 +945,27 @@ rem_port* INET_connect(const TEXT* name, } if (!setNoNagleOption(port)) - { gds__log("setsockopt: error setting TCP_NODELAY"); - goto err_close; - } - - setFastLoopbackOption(port); - - n = connect(port->port_handle, pai->ai_addr, pai->ai_addrlen); - if (n != -1) + else { - port->port_peer_name = host; - get_peer_info(port); - if (send_full(port, packet)) - goto exit_free; + setFastLoopbackOption(port); + + n = connect(port->port_handle, pai->ai_addr, pai->ai_addrlen); + if (n != -1) + { + port->port_peer_name = host; + get_peer_info(port); + if (send_full(port, packet)) + return port; + } } } else { // server - port = listener_socket(port, flag, pai); - goto exit_free; + return listener_socket(port, flag, pai); } -err_close: SOCLOSE(port->port_handle); } @@ -968,8 +975,6 @@ err_close: else inet_error(true, port, "listen", isc_net_connect_listen_err, 0); -exit_free: - freeaddrinfo(gai_result); return port; } From 926ec1b8303be5a2838d73824f99e929af703c44 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sun, 24 Nov 2019 00:03:45 +0000 Subject: [PATCH 114/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 4602b6b5f1..bcee6b5f29 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1651 + FORMAL BUILD NUMBER:1652 */ -#define PRODUCT_VER_STRING "4.0.0.1651" -#define FILE_VER_STRING "WI-T4.0.0.1651" -#define LICENSE_VER_STRING "WI-T4.0.0.1651" -#define FILE_VER_NUMBER 4, 0, 0, 1651 +#define PRODUCT_VER_STRING "4.0.0.1652" +#define FILE_VER_STRING "WI-T4.0.0.1652" +#define LICENSE_VER_STRING "WI-T4.0.0.1652" +#define FILE_VER_NUMBER 4, 0, 0, 1652 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1651" +#define FB_BUILD_NO "1652" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 7c27974401..45f7c07512 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1651 +BuildNum=1652 NowAt=`pwd` cd `dirname $0` From ba516f68a8f8854eb05f7cb31129e61f02d10a5d Mon Sep 17 00:00:00 2001 From: Dmitry Starodubov Date: Tue, 26 Nov 2019 12:01:41 +0300 Subject: [PATCH 115/274] Fixed CORE-6198: Wrong error checking for pread / pwrite calls --- src/jrd/os/posix/unix.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/jrd/os/posix/unix.cpp b/src/jrd/os/posix/unix.cpp index 47631b113e..b1c5ab06fb 100644 --- a/src/jrd/os/posix/unix.cpp +++ b/src/jrd/os/posix/unix.cpp @@ -543,7 +543,7 @@ void PIO_header(thread_db* tdbb, UCHAR* address, int length) Database* const dbb = tdbb->getDatabase(); int i; - FB_UINT64 bytes; + SINT64 bytes; PageSpace* pageSpace = dbb->dbb_page_manager.findPageSpace(DB_PAGE_SPACE); jrd_file* file = pageSpace->file; @@ -553,7 +553,7 @@ void PIO_header(thread_db* tdbb, UCHAR* address, int length) for (i = 0; i < IO_RETRY; i++) { - if ((bytes = os_utils::pread(file->fil_desc, address, length, 0)) == (FB_UINT64) -1) + if ((bytes = os_utils::pread(file->fil_desc, address, length, 0)) < 0) { if (SYSCALL_INTERRUPTED(errno)) continue; @@ -636,7 +636,7 @@ USHORT PIO_init_data(thread_db* tdbb, jrd_file* main_file, FbStatusVector* statu write_pages = leftPages; SLONG to_write = write_pages * dbb->dbb_page_size; - SLONG written; + SINT64 written; for (int r = 0; r < IO_RETRY; r++) { @@ -644,7 +644,7 @@ USHORT PIO_init_data(thread_db* tdbb, jrd_file* main_file, FbStatusVector* statu return false; if ((written = os_utils::pwrite(file->fil_desc, zero_buff, to_write, LSEEK_OFFSET_CAST offset)) == to_write) break; - if (written == (SLONG) -1 && !SYSCALL_INTERRUPTED(errno)) + if (written < 0 && !SYSCALL_INTERRUPTED(errno)) return unix_error("write", file, isc_io_write_err, status_vector); } @@ -751,7 +751,8 @@ bool PIO_read(thread_db* tdbb, jrd_file* file, BufferDesc* bdb, Ods::pag* page, * **************************************/ int i; - FB_UINT64 bytes, offset; + SINT64 bytes; + FB_UINT64 offset; if (file->fil_desc == -1) return unix_error("read", file, isc_io_read_err, status_vector); @@ -768,7 +769,7 @@ bool PIO_read(thread_db* tdbb, jrd_file* file, BufferDesc* bdb, Ods::pag* page, return false; if ((bytes = os_utils::pread(file->fil_desc, page, size, LSEEK_OFFSET_CAST offset)) == size) break; - if (bytes == -1U && !SYSCALL_INTERRUPTED(errno)) + if (bytes < 0 && !SYSCALL_INTERRUPTED(errno)) return unix_error("read", file, isc_io_read_err, status_vector); } @@ -809,8 +810,8 @@ bool PIO_write(thread_db* tdbb, jrd_file* file, BufferDesc* bdb, Ods::pag* page, * **************************************/ int i; - SLONG bytes; - FB_UINT64 offset; + SINT64 bytes; + FB_UINT64 offset; if (file->fil_desc == -1) return unix_error("write", file, isc_io_write_err, status_vector); @@ -827,7 +828,7 @@ bool PIO_write(thread_db* tdbb, jrd_file* file, BufferDesc* bdb, Ods::pag* page, return false; if ((bytes = os_utils::pwrite(file->fil_desc, page, size, LSEEK_OFFSET_CAST offset)) == size) break; - if (bytes == (SLONG) -1 && !SYSCALL_INTERRUPTED(errno)) + if (bytes < 0 && !SYSCALL_INTERRUPTED(errno)) return unix_error("write", file, isc_io_write_err, status_vector); } From f3536a70c498f0887d991ebca708b7a53019dcf1 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 27 Nov 2019 00:03:46 +0000 Subject: [PATCH 116/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index bcee6b5f29..b73e3fdd7a 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1652 + FORMAL BUILD NUMBER:1653 */ -#define PRODUCT_VER_STRING "4.0.0.1652" -#define FILE_VER_STRING "WI-T4.0.0.1652" -#define LICENSE_VER_STRING "WI-T4.0.0.1652" -#define FILE_VER_NUMBER 4, 0, 0, 1652 +#define PRODUCT_VER_STRING "4.0.0.1653" +#define FILE_VER_STRING "WI-T4.0.0.1653" +#define LICENSE_VER_STRING "WI-T4.0.0.1653" +#define FILE_VER_NUMBER 4, 0, 0, 1653 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1652" +#define FB_BUILD_NO "1653" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 45f7c07512..a014e0d4f3 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1652 +BuildNum=1653 NowAt=`pwd` cd `dirname $0` From f6a3653baa832667ab36b485bea10f03cea97d35 Mon Sep 17 00:00:00 2001 From: Roman Simakov Date: Wed, 27 Nov 2019 17:20:32 +0300 Subject: [PATCH 117/274] Now CREATE FUNCTION/PROCEDURE inside CREATE PACKAGE does not require CREATE FUNCTION/PROCEDURE privilege --- src/dsql/Nodes.h | 15 +++------------ src/jrd/scl.epp | 5 ++++- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/dsql/Nodes.h b/src/dsql/Nodes.h index aa6a4a3993..805c159534 100644 --- a/src/dsql/Nodes.h +++ b/src/dsql/Nodes.h @@ -200,20 +200,11 @@ public: if (dsqlScratch) dsqlScratch->setTransaction(transaction); - try - { - if (checkPermission(tdbb, transaction)) - tdbb->tdbb_flags |= TDBB_trusted_ddl; + const ULONG flag = checkPermission(tdbb, transaction) ? TDBB_trusted_ddl : 0; - execute(tdbb, dsqlScratch, transaction); - } - catch (...) - { - tdbb->tdbb_flags &= ~TDBB_trusted_ddl; - throw; - } + Firebird::AutoSetRestoreFlag trustedDdlFlag(&tdbb->tdbb_flags, flag, true); - tdbb->tdbb_flags &= ~TDBB_trusted_ddl; + execute(tdbb, dsqlScratch, transaction); } virtual DdlNode* dsqlPass(DsqlCompilerScratch* dsqlScratch) diff --git a/src/jrd/scl.epp b/src/jrd/scl.epp index e8bf7ac24f..b1629fe9f0 100644 --- a/src/jrd/scl.epp +++ b/src/jrd/scl.epp @@ -295,7 +295,10 @@ void SCL_check_create_access(thread_db* tdbb, int type) Jrd::Attachment* const attachment = tdbb->getAttachment(); // Allow the locksmith any access to database - if (attachment->locksmith(tdbb, SystemPrivilege::MODIFY_ANY_OBJECT_IN_DATABASE)) + // TDBB_trusted_ddl flag may be set in CREATE PACKAGE node. I suppose + // if user has CREATE PACKAGE privilege we should not require his to have CREATE FUNCTION as well + if ((tdbb->tdbb_flags & TDBB_trusted_ddl) || + attachment->locksmith(tdbb, SystemPrivilege::MODIFY_ANY_OBJECT_IN_DATABASE)) return; const SecurityClass::flags_t obj_mask = SCL_get_object_mask(type); From d5718b6a1c70b6afee54b9f12dcf1cc377fbaaee Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Thu, 28 Nov 2019 00:03:59 +0000 Subject: [PATCH 118/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index b73e3fdd7a..8ef03d3807 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1653 + FORMAL BUILD NUMBER:1654 */ -#define PRODUCT_VER_STRING "4.0.0.1653" -#define FILE_VER_STRING "WI-T4.0.0.1653" -#define LICENSE_VER_STRING "WI-T4.0.0.1653" -#define FILE_VER_NUMBER 4, 0, 0, 1653 +#define PRODUCT_VER_STRING "4.0.0.1654" +#define FILE_VER_STRING "WI-T4.0.0.1654" +#define LICENSE_VER_STRING "WI-T4.0.0.1654" +#define FILE_VER_NUMBER 4, 0, 0, 1654 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1653" +#define FB_BUILD_NO "1654" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index a014e0d4f3..0c20515ecf 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1653 +BuildNum=1654 NowAt=`pwd` cd `dirname $0` From f9d2dcb56a9b347146a131817ae80e8e580d70a7 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 29 Nov 2019 20:03:56 +0300 Subject: [PATCH 119/274] Added cloop-generated file to clean target --- builds/posix/Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builds/posix/Makefile.in b/builds/posix/Makefile.in index c34bca0b7e..d8ce881eb7 100644 --- a/builds/posix/Makefile.in +++ b/builds/posix/Makefile.in @@ -712,7 +712,7 @@ clean_vers: $(RM) *.vers clean_misc: - $(RM) ods.txt odstest* security.tmp test.header.txt + $(RM) ods.txt odstest* security.tmp test.header.txt $(API_PAS_FILE) ifeq ($(EDITLINE_FLG),Y) ifeq ($(STD_EDITLINE),false) From a420d97a74d4c2a4fd42ea63cea575e6b9b2a9b6 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 30 Nov 2019 00:03:56 +0000 Subject: [PATCH 120/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 8ef03d3807..e6615f4206 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1654 + FORMAL BUILD NUMBER:1655 */ -#define PRODUCT_VER_STRING "4.0.0.1654" -#define FILE_VER_STRING "WI-T4.0.0.1654" -#define LICENSE_VER_STRING "WI-T4.0.0.1654" -#define FILE_VER_NUMBER 4, 0, 0, 1654 +#define PRODUCT_VER_STRING "4.0.0.1655" +#define FILE_VER_STRING "WI-T4.0.0.1655" +#define LICENSE_VER_STRING "WI-T4.0.0.1655" +#define FILE_VER_NUMBER 4, 0, 0, 1655 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1654" +#define FB_BUILD_NO "1655" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 0c20515ecf..24deae3b5f 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1654 +BuildNum=1655 NowAt=`pwd` cd `dirname $0` From b1b93bb1ec6e055ad14d6c99588504e227294c93 Mon Sep 17 00:00:00 2001 From: Dmitry Yemanov Date: Wed, 13 Nov 2019 15:04:25 +0300 Subject: [PATCH 121/274] Fixed ugly initialization of the shared memory object + fixed potential problems with 64-bit transaction IDs + minor refactoring --- src/jrd/tpc.cpp | 224 +++++++++++++++++++++++--------------------- src/jrd/tpc_proto.h | 24 ++--- 2 files changed, 128 insertions(+), 120 deletions(-) diff --git a/src/jrd/tpc.cpp b/src/jrd/tpc.cpp index 9a9b4bdc85..1a7d6af444 100644 --- a/src/jrd/tpc.cpp +++ b/src/jrd/tpc.cpp @@ -48,30 +48,27 @@ void TipCache::MemoryInitializer::mutexBug(int osErrorCode, const char* text) bool TipCache::GlobalTpcInitializer::initialize(Firebird::SharedMemoryBase* sm, bool initFlag) { - m_cache->m_tpcHeader = static_cast*>(sm); + GlobalTpcHeader* header = static_cast(sm->sh_mem_header); if (!initFlag) { - m_cache->initTransactionsPerBlock(); - m_cache->mapInventoryPages(); + m_cache->initTransactionsPerBlock(header->tpc_block_size); + m_cache->mapInventoryPages(header); return true; } thread_db* tdbb = JRD_get_thread_data(); - - GlobalTpcHeader* header = m_cache->m_tpcHeader->getHeader(); + Database* dbb = tdbb->getDatabase(); // Initialize the shared data header - header->mhb_type = SharedMemoryBase::SRAM_TPC_HEADER; - header->mhb_version = TPC_VERSION; - header->mhb_timestamp = TimeStamp::getCurrentTimeStamp().value(); + header->init(SharedMemoryBase::SRAM_TPC_HEADER, TPC_VERSION); header->latest_commit_number.store(CN_PREHISTORIC, std::memory_order_relaxed); header->latest_statement_id.store(0, std::memory_order_relaxed); - header->tpc_block_size = m_cache->m_dbb->dbb_config->getTipCacheBlockSize(); + header->tpc_block_size = dbb->dbb_config->getTipCacheBlockSize(); - m_cache->initTransactionsPerBlock(); - m_cache->loadInventoryPages(tdbb); + m_cache->initTransactionsPerBlock(header->tpc_block_size); + m_cache->loadInventoryPages(tdbb, header); return true; } @@ -84,9 +81,7 @@ bool TipCache::SnapshotsInitializer::initialize(Firebird::SharedMemoryBase* sm, SnapshotList* header = static_cast(sm->sh_mem_header); // Initialize the shared data header - header->mhb_type = SharedMemoryBase::SRAM_TPC_SNAPSHOTS; - header->mhb_version = TPC_VERSION; - header->mhb_timestamp = TimeStamp::getCurrentTimeStamp().value(); + header->init(SharedMemoryBase::SRAM_TPC_SNAPSHOTS, TPC_VERSION); header->slots_used.store(0, std::memory_order_relaxed); header->min_free_slot = 0; @@ -104,9 +99,7 @@ bool TipCache::MemBlockInitializer::initialize(Firebird::SharedMemoryBase* sm, b TransactionStatusBlock* header = static_cast(sm->sh_mem_header); // Initialize the shared data header - header->mhb_type = SharedMemoryBase::SRAM_TPC_BLOCK; - header->mhb_version = TPC_VERSION; - header->mhb_timestamp = TimeStamp::getCurrentTimeStamp().value(); + header->init(SharedMemoryBase::SRAM_TPC_BLOCK, TPC_VERSION); memset(header->data, 0, sm->sh_mem_length_mapped - offsetof(TransactionStatusBlock, data[0])); @@ -116,7 +109,7 @@ bool TipCache::MemBlockInitializer::initialize(Firebird::SharedMemoryBase* sm, b } TipCache::TipCache(Database* dbb) - : m_dbb(dbb), m_tpcHeader(NULL), m_snapshots(NULL), m_transactionsPerBlock(0), + : m_tpcHeader(NULL), m_snapshots(NULL), m_transactionsPerBlock(0), globalTpcInitializer(this), snapshotsInitializer(this), memBlockInitializer(this), m_blocks_memory(*dbb->dbb_permanent) { @@ -175,8 +168,9 @@ void TipCache::finalizeTpc(thread_db* tdbb) CommitNumber TipCache::cacheState(TraNumber number) { fb_assert(m_tpcHeader); + GlobalTpcHeader* header = m_tpcHeader->getHeader(); - TraNumber oldest = m_tpcHeader->getHeader()->oldest_transaction.load(std::memory_order_relaxed); + TraNumber oldest = header->oldest_transaction.load(std::memory_order_relaxed); if (number < oldest) return CN_PREHISTORIC; @@ -186,9 +180,9 @@ CommitNumber TipCache::cacheState(TraNumber number) // TransactionStatusBlock granularity and iterate over transactions // directly. But since this function is not really called too frequently, // it should not matter and we leave interface "as is" for now. - int blockNumber = number / m_transactionsPerBlock; - int offset = number % m_transactionsPerBlock; - TransactionStatusBlock* block = getTransactionStatusBlock(blockNumber); + TpcBlockNumber blockNumber = number / m_transactionsPerBlock; + ULONG offset = number % m_transactionsPerBlock; + TransactionStatusBlock* block = getTransactionStatusBlock(header, blockNumber); // This should not really happen ever fb_assert(block); @@ -206,6 +200,8 @@ CommitNumber TipCache::cacheState(TraNumber number) void TipCache::initializeTpc(thread_db *tdbb) { + Database* dbb = tdbb->getDatabase(); + // Initialization can only be called on a TipCache that is not initialized fb_assert(!m_transactionsPerBlock); @@ -216,21 +212,15 @@ void TipCache::initializeTpc(thread_db *tdbb) ERR_bugcheck_msg("Unable to obtain TPC lock (PR)"); string fileName; - fileName.printf(TPC_HDR_FILE, m_dbb->getUniqueFileId().c_str()); + try { - // XXX: This will indirectly set m_tpcHeader and store the pointer to an object - // This is quite dirty C++ and will need to be fixed eventually - fb_assert(!m_tpcHeader); - - FB_NEW_POOL(*m_dbb->dbb_permanent) SharedMemory( + fileName.printf(TPC_HDR_FILE, dbb->getUniqueFileId().c_str()); + m_tpcHeader = FB_NEW_POOL(*dbb->dbb_permanent) SharedMemory( fileName.c_str(), sizeof(GlobalTpcHeader), &globalTpcInitializer); - - fb_assert(m_tpcHeader); } catch (const Exception& ex) { - m_tpcHeader = NULL; // This is to prevent double free due to the hack above iscLogException("TPC: Cannot initialize the shared memory region (header)", ex); LCK_release(tdbb, &lock); @@ -240,11 +230,11 @@ void TipCache::initializeTpc(thread_db *tdbb) fb_assert(m_tpcHeader->getHeader()->mhb_version == TPC_VERSION); - fileName.printf(SNAPSHOTS_FILE, m_dbb->getUniqueFileId().c_str()); try { - m_snapshots = FB_NEW_POOL(*m_dbb->dbb_permanent) SharedMemory( - fileName.c_str(), m_dbb->dbb_config->getSnapshotsMemSize(), &snapshotsInitializer); + fileName.printf(SNAPSHOTS_FILE, dbb->getUniqueFileId().c_str()); + m_snapshots = FB_NEW_POOL(*dbb->dbb_permanent) SharedMemory( + fileName.c_str(), dbb->dbb_config->getSnapshotsMemSize(), &snapshotsInitializer); } catch (const Exception& ex) { @@ -260,23 +250,23 @@ void TipCache::initializeTpc(thread_db *tdbb) LCK_release(tdbb, &lock); } -void TipCache::initTransactionsPerBlock() +void TipCache::initTransactionsPerBlock(ULONG blockSize) { if (m_transactionsPerBlock) return; const ULONG dataOffset = static_cast(offsetof(TransactionStatusBlock, data[0])); - m_transactionsPerBlock = - (m_tpcHeader->getHeader()->tpc_block_size - dataOffset) / sizeof(CommitNumber); + m_transactionsPerBlock = (blockSize - dataOffset) / sizeof(CommitNumber); } -void TipCache::loadInventoryPages(thread_db* tdbb) +void TipCache::loadInventoryPages(thread_db* tdbb, GlobalTpcHeader* header) { // check the header page for the oldest and newest transaction numbers #ifdef SUPERSERVER_V2 - const TraNumber top = m_dbb->dbb_next_transaction; - const TraNumber hdr_oldest = m_dbb->dbb_oldest_transaction; + Database* dbb = tdbb->getDatabase(); + const TraNumber top = dbb->dbb_next_transaction; + const TraNumber hdr_oldest = dbb->dbb_oldest_transaction; #else WIN window(HEADER_PAGE_NUMBER); const Ods::header_page* header_page = (Ods::header_page*) CCH_FETCH(tdbb, &window, LCK_read, pag_header); @@ -286,7 +276,6 @@ void TipCache::loadInventoryPages(thread_db* tdbb) CCH_RELEASE(tdbb, &window); #endif - GlobalTpcHeader* header = m_tpcHeader->getHeader(); header->oldest_transaction.store(hdr_oldest_transaction, std::memory_order_relaxed); header->latest_attachment_id.store(hdr_attachment_id, std::memory_order_relaxed); header->latest_transaction_id.store(hdr_next_transaction, std::memory_order_relaxed); @@ -309,9 +298,9 @@ void TipCache::loadInventoryPages(thread_db* tdbb) static const CommitNumber init_state_mapping[4] = {CN_ACTIVE, CN_LIMBO, CN_DEAD, CN_PREHISTORIC}; - int blockNumber = hdr_oldest_transaction / m_transactionsPerBlock; - int transOffset = hdr_oldest_transaction % m_transactionsPerBlock; - TransactionStatusBlock* statusBlock = getTransactionStatusBlock(blockNumber); + TpcBlockNumber blockNumber = hdr_oldest_transaction / m_transactionsPerBlock; + ULONG transOffset = hdr_oldest_transaction % m_transactionsPerBlock; + TransactionStatusBlock* statusBlock = getTransactionStatusBlock(header, blockNumber); for (TraNumber t = hdr_oldest_transaction; ; ) { @@ -330,43 +319,40 @@ void TipCache::loadInventoryPages(thread_db* tdbb) { blockNumber++; transOffset = 0; - statusBlock = getTransactionStatusBlock(blockNumber); + statusBlock = getTransactionStatusBlock(header, blockNumber); } } } -void TipCache::mapInventoryPages() +void TipCache::mapInventoryPages(GlobalTpcHeader* header) { - GlobalTpcHeader* header = m_tpcHeader->getHeader(); - int blockNumber = header->oldest_transaction / m_transactionsPerBlock; - const int lastNumber = header->latest_transaction_id / m_transactionsPerBlock; + TpcBlockNumber blockNumber = header->oldest_transaction / m_transactionsPerBlock; + const TpcBlockNumber lastNumber = header->latest_transaction_id / m_transactionsPerBlock; for (; blockNumber <= lastNumber; blockNumber++) - getTransactionStatusBlock(blockNumber); + getTransactionStatusBlock(header, blockNumber); } -TipCache::StatusBlockData::StatusBlockData(thread_db* tdbb, TipCache* tipCache, int blkNumber) +TipCache::StatusBlockData::StatusBlockData(thread_db* tdbb, TipCache* tipCache, ULONG blockSize, TpcBlockNumber blkNumber) : blockNumber(blkNumber), memory(NULL), - existenceLock(tdbb, sizeof(SLONG), LCK_tpc_block, this, tpc_block_blocking_ast), + existenceLock(tdbb, sizeof(TpcBlockNumber), LCK_tpc_block, this, tpc_block_blocking_ast), cache(tipCache) { - Database* dbb = tipCache->m_dbb; - MemoryPool* pool = dbb->dbb_permanent; + Database* dbb = tdbb->getDatabase(); - memory = NULL; - cache = tipCache; existenceLock.setKey(blockNumber); if (!LCK_lock(tdbb, &existenceLock, LCK_SR, LCK_WAIT)) ERR_bugcheck_msg("Unable to obtain memory block lock"); string fileName; - fileName.printf(TPC_BLOCK_FILE, cache->m_dbb->getUniqueFileId().c_str(), blockNumber); + fileName.printf(TPC_BLOCK_FILE, dbb->getUniqueFileId().c_str(), blockNumber); + try { - memory = FB_NEW_POOL(*pool) Firebird::SharedMemory( - fileName.c_str(), cache->m_tpcHeader->getHeader()->tpc_block_size, + memory = FB_NEW_POOL(*dbb->dbb_permanent) Firebird::SharedMemory( + fileName.c_str(), blockSize, &cache->memBlockInitializer, true); } catch (const Exception& ex) @@ -398,21 +384,22 @@ void TipCache::StatusBlockData::clear(thread_db* tdbb) LCK_release(tdbb, &existenceLock); } -TipCache::TransactionStatusBlock* TipCache::createTransactionStatusBlock(int blockNumber) +TipCache::TransactionStatusBlock* TipCache::createTransactionStatusBlock(ULONG blockSize, TpcBlockNumber blockNumber) { fb_assert(m_sync_status.ourExclusiveLock()); thread_db* tdbb = JRD_get_thread_data(); Database* dbb = tdbb->getDatabase(); - StatusBlockData* blockData = FB_NEW_POOL(*dbb->dbb_permanent) StatusBlockData(tdbb, this, blockNumber); + StatusBlockData* blockData = FB_NEW_POOL(*dbb->dbb_permanent) + StatusBlockData(tdbb, this, blockSize, blockNumber); m_blocks_memory.add(blockData); return blockData->memory->getHeader(); } -TipCache::TransactionStatusBlock* TipCache::getTransactionStatusBlock(int blockNumber) +TipCache::TransactionStatusBlock* TipCache::getTransactionStatusBlock(GlobalTpcHeader* header, TpcBlockNumber blockNumber) { // This is a double-checked locking pattern. SyncLockGuard uses atomic ops internally and should be cheap TransactionStatusBlock* block = NULL; @@ -432,9 +419,9 @@ TipCache::TransactionStatusBlock* TipCache::getTransactionStatusBlock(int blockN else { // Check if block might be too old to be created. - TraNumber oldest = m_tpcHeader->getHeader()->oldest_transaction.load(std::memory_order_relaxed); - if (blockNumber >= static_cast(oldest / m_transactionsPerBlock)) - block = createTransactionStatusBlock(blockNumber); + TraNumber oldest = header->oldest_transaction.load(std::memory_order_relaxed); + if (blockNumber >= oldest / m_transactionsPerBlock) + block = createTransactionStatusBlock(header->tpc_block_size, blockNumber); } } return block; @@ -444,19 +431,22 @@ TraNumber TipCache::findStates(TraNumber minNumber, TraNumber maxNumber, ULONG m { // Can only be called on initialized TipCache fb_assert(m_tpcHeader); + GlobalTpcHeader* header = m_tpcHeader->getHeader(); TransactionStatusBlock* statusBlock; - int blockNumber, transOffset; + TpcBlockNumber blockNumber; + ULONG transOffset; + do { - TraNumber oldest = m_tpcHeader->getHeader()->oldest_transaction.load(std::memory_order_relaxed); + TraNumber oldest = header->oldest_transaction.load(std::memory_order_relaxed); if (minNumber < oldest) minNumber = oldest; blockNumber = minNumber / m_transactionsPerBlock; transOffset = minNumber % m_transactionsPerBlock; - statusBlock = getTransactionStatusBlock(blockNumber); + statusBlock = getTransactionStatusBlock(header, blockNumber); } while (!statusBlock); for (TraNumber t = minNumber; ; ) @@ -498,25 +488,26 @@ TraNumber TipCache::findStates(TraNumber minNumber, TraNumber maxNumber, ULONG m { blockNumber++; transOffset = 0; - statusBlock = getTransactionStatusBlock(blockNumber); + statusBlock = getTransactionStatusBlock(header, blockNumber); } } return 0; } -CommitNumber TipCache::setState(TraNumber number, SSHORT state) +CommitNumber TipCache::setState(TraNumber number, int state) { // Can only be called on initialized TipCache fb_assert(m_tpcHeader); + GlobalTpcHeader* header = m_tpcHeader->getHeader(); // over large number of operations, if our callers are made aware of // TransactionStatusBlock granularity and iterate over transactions // directly. But since this function is not really called too frequently, // it should not matter and we leave interface "as is" for now. - int blockNumber = number / m_transactionsPerBlock; - int offset = number % m_transactionsPerBlock; - TransactionStatusBlock* block = getTransactionStatusBlock(blockNumber); + TpcBlockNumber blockNumber = number / m_transactionsPerBlock; + ULONG offset = number % m_transactionsPerBlock; + TransactionStatusBlock* block = getTransactionStatusBlock(header, blockNumber); // This should not really happen if (!block) @@ -539,7 +530,7 @@ CommitNumber TipCache::setState(TraNumber number, SSHORT state) fb_assert(oldStateCn == CN_ACTIVE || oldStateCn == CN_LIMBO); // Generate new commit number - CommitNumber newCommitNumber = m_tpcHeader->getHeader()->latest_commit_number++ + 1; + CommitNumber newCommitNumber = header->latest_commit_number++ + 1; statePtr->store(newCommitNumber, std::memory_order_relaxed); return newCommitNumber; @@ -623,12 +614,13 @@ void TipCache::updateOldestTransaction(thread_db *tdbb, TraNumber oldest, TraNum { // Can only be called on initialized TipCache fb_assert(m_tpcHeader); + GlobalTpcHeader* header = m_tpcHeader->getHeader(); TraNumber oldestNew = MIN(oldest, oldestSnapshot); - TraNumber oldestNow = m_tpcHeader->getHeader()->oldest_transaction.load(std::memory_order_relaxed); + TraNumber oldestNow = header->oldest_transaction.load(std::memory_order_relaxed); if (oldestNew > oldestNow) { - m_tpcHeader->getHeader()->oldest_transaction.store(oldestNew, std::memory_order_relaxed); + header->oldest_transaction.store(oldestNew, std::memory_order_relaxed); releaseSharedMemory(tdbb, oldestNow, oldestNew); } } @@ -637,27 +629,30 @@ int TipCache::tpc_block_blocking_ast(void* arg) { StatusBlockData* data = static_cast(arg); - Database* const dbb = data->existenceLock.lck_dbb; - AsyncContextHolder tdbb(dbb, FB_FUNCTION, &data->existenceLock); + Database* dbb = data->existenceLock.lck_dbb; + AsyncContextHolder tdbb(dbb, FB_FUNCTION); TipCache* cache = data->cache; - TraNumber oldest = cache->m_tpcHeader->getHeader()->oldest_transaction.load(std::memory_order_relaxed); + TraNumber oldest = + cache->m_tpcHeader->getHeader()->oldest_transaction.load(std::memory_order_relaxed); // Release shared memory data->clear(tdbb); // Check if there is a bug in cleanup code and we were requested to // release memory that might be in use - if (data->blockNumber >= static_cast(oldest / cache->m_transactionsPerBlock)) + if (data->blockNumber >= oldest / cache->m_transactionsPerBlock) ERR_bugcheck_msg("Incorrect attempt to release shared memory"); return 0; } -void TipCache::releaseSharedMemory(thread_db *tdbb, TraNumber oldest_old, TraNumber oldest_new) +void TipCache::releaseSharedMemory(thread_db* tdbb, TraNumber oldest_old, TraNumber oldest_new) { - int lastInterestingBlockNumber = oldest_new / m_transactionsPerBlock; + Database* dbb = tdbb->getDatabase(); + + TpcBlockNumber lastInterestingBlockNumber = oldest_new / m_transactionsPerBlock; // If we didn't cross block boundary - there is nothing to do. // Note that due to the fuzziness of our caller's memory access to variables @@ -666,16 +661,17 @@ void TipCache::releaseSharedMemory(thread_db *tdbb, TraNumber oldest_old, TraNum if (oldest_old / m_transactionsPerBlock == lastInterestingBlockNumber) return; - // Populate array of blocks that might be unmapped and deleted + // Populate array of blocks that might be unmapped and deleted. // We scan for blocks to clean up in descending order, but delete them in // ascending order to ensure for robust operation. string fileName; - Firebird::Array blocksToCleanup; - for (int blockNumber = lastInterestingBlockNumber - SAFETY_GAP_BLOCKS - 1; - blockNumber >= 0; - blockNumber--) + Firebird::HalfStaticArray blocksToCleanup; + + for (TpcBlockNumber cleanupCounter = lastInterestingBlockNumber - SAFETY_GAP_BLOCKS; + cleanupCounter; cleanupCounter--) { - fileName.printf(TPC_BLOCK_FILE, m_dbb->getUniqueFileId().c_str(), blockNumber); + TpcBlockNumber blockNumber = cleanupCounter - 1; + fileName.printf(TPC_BLOCK_FILE, dbb->getUniqueFileId().c_str(), blockNumber); TEXT expanded_filename[MAXPATHLEN]; iscPrefixLock(expanded_filename, fileName.c_str(), false); @@ -691,24 +687,23 @@ void TipCache::releaseSharedMemory(thread_db *tdbb, TraNumber oldest_old, TraNum return; SyncLockGuard sync(&m_sync_status, SYNC_EXCLUSIVE, "TipCache::releaseSharedMemory"); - for (int* itr = blocksToCleanup.end() - 1; itr >= blocksToCleanup.begin(); itr--) + while (blocksToCleanup.hasData()) { - int blockNumber = *itr; + TpcBlockNumber blockNumber = blocksToCleanup.pop(); if (m_blocks_memory.locate(blockNumber)) { - StatusBlockData* cur = m_blocks_memory.current(); + StatusBlockData* block = m_blocks_memory.current(); m_blocks_memory.fastRemove(); - - delete cur; + delete block; } // Signal other processes to release resources - Lock temp(tdbb, sizeof(SLONG), LCK_tpc_block); + Lock temp(tdbb, sizeof(TpcBlockNumber), LCK_tpc_block); temp.setKey(blockNumber); if (!LCK_lock(tdbb, &temp, LCK_EX, LCK_WAIT)) { - gds__log("TPC BUG: Unable to obtain cleanup lock for block %d. Please report this error to developers", *itr); + gds__log("TPC BUG: Unable to obtain cleanup lock for block %" UQUADFORMAT, blockNumber); fb_assert(false); break; } @@ -771,6 +766,7 @@ void TipCache::remapSnapshots(bool sync) fb_assert(m_tpcHeader); SnapshotList* snapshots = m_snapshots->getHeader(); + if (snapshots->slots_allocated.load(std::memory_order_acquire) != (m_snapshots->sh_mem_length_mapped - offsetof(SnapshotList, slots[0])) / sizeof(SnapshotData)) { @@ -795,6 +791,8 @@ SnapshotHandle TipCache::beginSnapshot(thread_db* tdbb, AttNumber attachmentId, { // Can only be called on initialized TipCache fb_assert(m_tpcHeader); + GlobalTpcHeader* header = m_tpcHeader->getHeader(); + fb_assert(attachmentId); // Lock mutex @@ -803,9 +801,10 @@ SnapshotHandle TipCache::beginSnapshot(thread_db* tdbb, AttNumber attachmentId, // Remap snapshot list if it has been grown by someone else remapSnapshots(false); + SnapshotList* snapshots = m_snapshots->getHeader(); + if (commitNumber != 0) { - SnapshotList* snapshots = m_snapshots->getHeader(); ULONG slotsUsed = snapshots->slots_used.load(std::memory_order_relaxed); bool found = false; @@ -826,13 +825,13 @@ SnapshotHandle TipCache::beginSnapshot(thread_db* tdbb, AttNumber attachmentId, SnapshotHandle slotNumber = allocateSnapshotSlot(); // Note, that allocateSnapshotSlot might remap memory and thus invalidate pointers - SnapshotList* snapshots = m_snapshots->getHeader(); + snapshots = m_snapshots->getHeader(); // Store snapshot commit number and return handle SnapshotData* slot = snapshots->slots + slotNumber; if (commitNumber == 0) - commitNumber = m_tpcHeader->getHeader()->latest_commit_number.load(std::memory_order_acquire); + commitNumber = header->latest_commit_number.load(std::memory_order_acquire); slot->snapshot.store(commitNumber, std::memory_order_release); @@ -876,6 +875,7 @@ void TipCache::endSnapshot(thread_db* tdbb, SnapshotHandle handle, AttNumber att { // Can only be called on initialized TipCache fb_assert(m_tpcHeader); + GlobalTpcHeader* header = m_tpcHeader->getHeader(); // Lock mutex SharedMutexGuard guard(m_snapshots); @@ -898,27 +898,28 @@ void TipCache::endSnapshot(thread_db* tdbb, SnapshotHandle handle, AttNumber att deallocateSnapshotSlot(handle); // Increment release event count - m_tpcHeader->getHeader()->snapshot_release_count++; + header->snapshot_release_count++; } void TipCache::updateActiveSnapshots(thread_db* tdbb, ActiveSnapshots* activeSnapshots) { // Can only be called on initialized TipCache fb_assert(m_tpcHeader); + GlobalTpcHeader* header = m_tpcHeader->getHeader(); + fb_assert(activeSnapshots); // This function is quite tricky as it reads snapshots list without locks (using atomics) + SnapshotList* snapshots = m_snapshots->getHeader(); + if (activeSnapshots->m_lastCommit == CN_ACTIVE) { // Slow path. Initialization. - GlobalTpcHeader* header = m_tpcHeader->getHeader(); activeSnapshots->m_lastCommit = header->latest_commit_number.load(std::memory_order_acquire); activeSnapshots->m_releaseCount = header->snapshot_release_count.load(std::memory_order_relaxed); - SnapshotList* snapshots = m_snapshots->getHeader(); - // If new slots are allocated past this value - we don't care as we preserved // lastCommit and new snapshots will have numbers >= lastCommit and we don't // GC them anyways @@ -982,9 +983,6 @@ void TipCache::updateActiveSnapshots(thread_db* tdbb, ActiveSnapshots* activeSna { // Fast path. Quick update. - GlobalTpcHeader* header = m_tpcHeader->getHeader(); - SnapshotList* snapshots = m_snapshots->getHeader(); - // Update m_lastCommit unconditionally, to prevent active snapshots list from // stalling when there is no snapshots created\released since last update. // Stalled list of active snapshots could stop intermediate garbage collection @@ -1033,10 +1031,11 @@ TraNumber TipCache::generateTransactionId() { // Can only be called on initialized TipCache fb_assert(m_tpcHeader); + GlobalTpcHeader* header = m_tpcHeader->getHeader(); // No barrier here, because inconsistency in transaction number order does not matter // for read-only databases where this function is used. - TraNumber transaction_id = m_tpcHeader->getHeader()->latest_transaction_id++ + 1; + TraNumber transaction_id = header->latest_transaction_id++ + 1; return transaction_id; } @@ -1044,10 +1043,11 @@ AttNumber TipCache::generateAttachmentId() { // Can only be called on initialized TipCache fb_assert(m_tpcHeader); + GlobalTpcHeader* header = m_tpcHeader->getHeader(); // No barrier here, because attachment id order does not generally matter // especially for read-only databases where this function is used. - AttNumber attachment_id = m_tpcHeader->getHeader()->latest_attachment_id++ + 1; + AttNumber attachment_id = header->latest_attachment_id++ + 1; return attachment_id; } @@ -1055,9 +1055,10 @@ StmtNumber TipCache::generateStatementId() { // Can only be called on initialized TipCache fb_assert(m_tpcHeader); + GlobalTpcHeader* header = m_tpcHeader->getHeader(); // No barrier here, because statement id order does not generally matter - StmtNumber statement_id = m_tpcHeader->getHeader()->latest_statement_id++ + 1; + StmtNumber statement_id = header->latest_statement_id++ + 1; return statement_id; } @@ -1068,13 +1069,18 @@ StmtNumber TipCache::generateStatementId() void TipCache::assignLatestAttachmentId(AttNumber number) { + // Can only be called on initialized TipCache + fb_assert(m_tpcHeader); + GlobalTpcHeader* header = m_tpcHeader->getHeader(); + // XXX: there is no paired acquire because value assigned here is not really used for now - m_tpcHeader->getHeader()->latest_attachment_id.store(number, std::memory_order_release); + header->latest_attachment_id.store(number, std::memory_order_release); } int TPC_snapshot_state(thread_db* tdbb, TraNumber number) { TipCache* cache = tdbb->getDatabase()->dbb_tip_cache; + // This function might be called early during database initialization when // TIP cache does not exist yet. if (!cache) diff --git a/src/jrd/tpc_proto.h b/src/jrd/tpc_proto.h index 14e5a83c7e..f03b603d00 100644 --- a/src/jrd/tpc_proto.h +++ b/src/jrd/tpc_proto.h @@ -87,7 +87,7 @@ public: // - tra_limbo -> tra_committed // - tra_limbo -> tra_dead // All other combinations are expressly forbidden with BUGCHECK error. - CommitNumber setState(TraNumber number, SSHORT state); + CommitNumber setState(TraNumber number, int state); // Compute most current state of a transaction. This is called when we encountered // a record produced by this transaction and need to know what to do with this record. @@ -197,18 +197,21 @@ private: typedef TransactionStatusBlock* PTransactionStatusBlock; + // Block number storage should match TraNumber to avoid unexpected overflows + typedef FB_UINT64 TpcBlockNumber; + class StatusBlockData { public: - StatusBlockData(Jrd::thread_db* tdbb, Jrd::TipCache* tipCache, int blkNumber); + StatusBlockData(Jrd::thread_db* tdbb, Jrd::TipCache* tipCache, ULONG blockSize, TpcBlockNumber blkNumber); ~StatusBlockData(); - int blockNumber; + TpcBlockNumber blockNumber; Firebird::SharedMemory* memory; Lock existenceLock; TipCache* cache; - inline static int& generate(const void* /*sender*/, StatusBlockData* item) + inline static TpcBlockNumber& generate(const void* /*sender*/, StatusBlockData* item) { return item->blockNumber; } @@ -251,7 +254,6 @@ private: static const ULONG TPC_VERSION = 1; static const int SAFETY_GAP_BLOCKS = 1; - Database* m_dbb; // final Firebird::SharedMemory* m_tpcHeader; // final Firebird::SharedMemory* m_snapshots; // final ULONG m_transactionsPerBlock; // final. When set, we assume TPC has been initialized. @@ -266,14 +268,14 @@ private: Firebird::SyncObject m_sync_status; - void initTransactionsPerBlock(); + void initTransactionsPerBlock(ULONG blockSize); // Returns block holding transaction state. // Returns NULL if requested block is too old and is no longer cached. - TransactionStatusBlock* getTransactionStatusBlock(int blockNumber); + TransactionStatusBlock* getTransactionStatusBlock(GlobalTpcHeader* header, TpcBlockNumber blockNumber); // Map shared memory for a block - TransactionStatusBlock* createTransactionStatusBlock(int blockNumber); + TransactionStatusBlock* createTransactionStatusBlock(ULONG blockSize, TpcBlockNumber blockNumber); // Release shared memory blocks, if possible. // We utilize one full MemoryBlock as a safety margin to account for possible @@ -281,9 +283,9 @@ private: void releaseSharedMemory(thread_db *tdbb, TraNumber oldest_old, TraNumber oldest_new); // Populate TIP cache from disk - void loadInventoryPages(thread_db *tdbb); + void loadInventoryPages(thread_db *tdbb, GlobalTpcHeader* header); // Init mapping for existing TIP blocks - void mapInventoryPages(); + void mapInventoryPages(GlobalTpcHeader* header); static int tpc_block_blocking_ast(void* arg); @@ -311,7 +313,7 @@ inline TraNumber TPC_find_states(thread_db* tdbb, TraNumber minNumber, TraNumber return tdbb->getDatabase()->dbb_tip_cache->findStates(minNumber, maxNumber, mask, state); } -inline void TPC_set_state(thread_db* tdbb, TraNumber number, SSHORT state) +inline void TPC_set_state(thread_db* tdbb, TraNumber number, int state) { tdbb->getDatabase()->dbb_tip_cache->setState(number, state); } From d8f90b0aec2b81aef15764277b9d6f94b89f7421 Mon Sep 17 00:00:00 2001 From: Dmitry Yemanov Date: Sun, 1 Dec 2019 11:48:36 +0300 Subject: [PATCH 122/274] Frontported my fixes + some shmem refactoring + minor changelog cleanup --- src/common/file_params.h | 8 ++- src/common/isc_s_proto.h | 13 ++++ src/common/isc_sync.cpp | 6 +- src/jrd/Monitoring.cpp | 110 +++++++++++++++++++----------- src/jrd/Monitoring.h | 11 +-- src/jrd/event.cpp | 53 ++++---------- src/jrd/event_proto.h | 4 +- src/jrd/replication/ChangeLog.cpp | 94 ++++++++++++++----------- src/jrd/replication/ChangeLog.h | 19 +++--- src/lock/lock.cpp | 58 ++++------------ src/lock/lock_proto.h | 4 +- 11 files changed, 196 insertions(+), 184 deletions(-) diff --git a/src/common/file_params.h b/src/common/file_params.h index 3b3a97b325..0cd4ecc1c4 100644 --- a/src/common/file_params.h +++ b/src/common/file_params.h @@ -37,11 +37,13 @@ static const char* const EVENT_FILE = "fb_event_%s"; static const char* const LOCK_FILE = "fb_lock_%s"; static const char* const MONITOR_FILE = "fb_monitor_%s"; +static const char* const REPL_FILE = "fb_repl_%s"; +static const char* const TPC_HDR_FILE = "fb_tpc_%s"; +static const char* const TPC_BLOCK_FILE = "fb_tpc_%s_%" UQUADFORMAT; +static const char* const SNAPSHOTS_FILE = "fb_snap_%s"; + static const char* const TRACE_FILE = "fb" COMMON_FILE_PREFIX "_trace"; static const char* const USER_MAP_FILE = "fb" COMMON_FILE_PREFIX "_user_mapping"; -static const char* const TPC_HDR_FILE = "fb" COMMON_FILE_PREFIX "_tpc_%s"; -static const char* const TPC_BLOCK_FILE = "fb" COMMON_FILE_PREFIX "_tpc_%s_%d"; -static const char* const SNAPSHOTS_FILE = "fb" COMMON_FILE_PREFIX "_snap_%s"; #ifdef UNIX static const char* const INIT_FILE = "fb_init"; diff --git a/src/common/isc_s_proto.h b/src/common/isc_s_proto.h index eca91dd885..63339915f2 100644 --- a/src/common/isc_s_proto.h +++ b/src/common/isc_s_proto.h @@ -339,8 +339,21 @@ private: void freeSem5(Sys5Semaphore* sem); #endif + bool justCreated() + { + if (sh_mem_just_created) + { + // complete initialization + sh_mem_just_created = false; + return true; + } + + return false; + } + private: IpcObject* sh_mem_callback; + bool sh_mem_just_created; #ifdef WIN_NT bool sh_mem_unlink; #endif diff --git a/src/common/isc_sync.cpp b/src/common/isc_sync.cpp index c2495c61f1..9ff13a952b 100644 --- a/src/common/isc_sync.cpp +++ b/src/common/isc_sync.cpp @@ -1965,9 +1965,9 @@ SharedMemoryBase::SharedMemoryBase(const TEXT* filename, ULONG length, IpcObject if (trunc_flag) FB_UNUSED(os_utils::ftruncate(mainLock->getFd(), length)); + sh_mem_just_created = true; if (callback->initialize(this, true)) { - #ifdef HAVE_SHARED_MUTEX_SECTION #ifdef USE_SYS5SEMAPHORE @@ -2050,7 +2050,7 @@ SharedMemoryBase::SharedMemoryBase(const TEXT* filename, ULONG length, IpcObject #ifdef BUGGY_LINUX_MUTEX && (state != ENOTSUP || bugFlag) #endif - ) + ) { iscLogStatus("Pthread Error", (Arg::Gds(isc_sys_request) << "pthread_mutex_init" << Arg::Unix(state)).value()); @@ -2089,6 +2089,7 @@ SharedMemoryBase::SharedMemoryBase(const TEXT* filename, ULONG length, IpcObject } else { + sh_mem_just_created = false; if (callback->initialize(this, false)) { if (!mainLock->setlock(&statusVector, FileLock::FLM_SHARED)) @@ -2417,6 +2418,7 @@ SharedMemoryBase::SharedMemoryBase(const TEXT* filename, ULONG length, IpcObject sh_mem_hdr_address = header_address; strcpy(sh_mem_name, filename); + sh_mem_just_created = init_flag; sh_mem_callback->initialize(this, init_flag); if (init_flag) diff --git a/src/jrd/Monitoring.cpp b/src/jrd/Monitoring.cpp index f20f9f271a..ebf8083f5b 100644 --- a/src/jrd/Monitoring.cpp +++ b/src/jrd/Monitoring.cpp @@ -124,14 +124,33 @@ bool MonitoringTableScan::retrieveRecord(thread_db* tdbb, jrd_rel* relation, // MonitoringData class MonitoringData::MonitoringData(const Database* dbb) + : PermanentStorage(*dbb->dbb_permanent), + m_dbId(getPool(), dbb->getUniqueFileId()) { - string name; - name.printf(MONITOR_FILE, dbb->getUniqueFileId().c_str()); + initSharedFile(); +} + + +MonitoringData::~MonitoringData() +{ + Guard guard(this); + + if (m_sharedMemory->getHeader() && + m_sharedMemory->getHeader()->used == alignOffset(sizeof(Header))) + { + m_sharedMemory->removeMapFile(); + } +} + + +void MonitoringData::initSharedFile() +{ + PathName name; + name.printf(MONITOR_FILE, m_dbId.c_str()); - Arg::StatusVector statusVector; try { - shared_memory.reset(FB_NEW_POOL(*dbb->dbb_permanent) + m_sharedMemory.reset(FB_NEW_POOL(getPool()) SharedMemory(name.c_str(), DEFAULT_SIZE, this)); } catch (const Exception& ex) @@ -140,33 +159,44 @@ MonitoringData::MonitoringData(const Database* dbb) throw; } - fb_assert(shared_memory->getHeader()->mhb_header_version == MemoryHeader::HEADER_VERSION); - fb_assert(shared_memory->getHeader()->mhb_version == MONITOR_VERSION); -} - - -MonitoringData::~MonitoringData() -{ - Guard guard(this); - - if (shared_memory->getHeader()->used == alignOffset(sizeof(Header))) - shared_memory->removeMapFile(); + fb_assert(m_sharedMemory->getHeader()->mhb_type == SharedMemoryBase::SRAM_DATABASE_SNAPSHOT); + fb_assert(m_sharedMemory->getHeader()->mhb_header_version == MemoryHeader::HEADER_VERSION); + fb_assert(m_sharedMemory->getHeader()->mhb_version == MONITOR_VERSION); } void MonitoringData::acquire() { - shared_memory->mutexLock(); + m_sharedMemory->mutexLock(); - if (shared_memory->getHeader()->allocated > shared_memory->sh_mem_length_mapped) + while (m_sharedMemory->getHeader()->used == alignOffset(sizeof(Header))) + { + if (m_sharedMemory->justCreated()) + break; + + // Someone is going to delete shared file? Reattach. + m_sharedMemory->mutexUnlock(); + m_sharedMemory.reset(); + + Thread::yield(); + + initSharedFile(); + m_sharedMemory->mutexLock(); + } + + fb_assert(!m_sharedMemory->justCreated()); + + if (m_sharedMemory->getHeader()->allocated > m_sharedMemory->sh_mem_length_mapped) { #ifdef HAVE_OBJECT_MAP FbLocalStatus statusVector; - if (!shared_memory->remapFile(&statusVector, shared_memory->getHeader()->allocated, false)) + if (!m_sharedMemory->remapFile(&statusVector, m_sharedMemory->getHeader()->allocated, false)) { + m_sharedMemory->mutexUnlock(); status_exception::raise(&statusVector); } #else + m_sharedMemory->mutexUnlock(); status_exception::raise(Arg::Gds(isc_montabexh)); #endif } @@ -175,7 +205,7 @@ void MonitoringData::acquire() void MonitoringData::release() { - shared_memory->mutexUnlock(); + m_sharedMemory->mutexUnlock(); } @@ -185,9 +215,9 @@ void MonitoringData::read(const char* user_name, TempSpace& temp) // Copy data of all permitted sessions - for (ULONG offset = alignOffset(sizeof(Header)); offset < shared_memory->getHeader()->used;) + for (ULONG offset = alignOffset(sizeof(Header)); offset < m_sharedMemory->getHeader()->used;) { - UCHAR* const ptr = (UCHAR*) shared_memory->getHeader() + offset; + UCHAR* const ptr = (UCHAR*) m_sharedMemory->getHeader() + offset; const Element* const element = (Element*) ptr; const ULONG length = alignOffset(sizeof(Element) + element->length); @@ -204,19 +234,19 @@ void MonitoringData::read(const char* user_name, TempSpace& temp) ULONG MonitoringData::setup(AttNumber att_id, const char* user_name) { - const ULONG offset = alignOffset(shared_memory->getHeader()->used); - const ULONG delta = offset + sizeof(Element) - shared_memory->getHeader()->used; + const ULONG offset = alignOffset(m_sharedMemory->getHeader()->used); + const ULONG delta = offset + sizeof(Element) - m_sharedMemory->getHeader()->used; ensureSpace(delta); // Prepare for writing new data at the tail - UCHAR* const ptr = (UCHAR*) shared_memory->getHeader() + offset; + UCHAR* const ptr = (UCHAR*) m_sharedMemory->getHeader() + offset; Element* const element = (Element*) ptr; element->attId = att_id; snprintf(element->userName, sizeof(element->userName), "%s", user_name); element->length = 0; - shared_memory->getHeader()->used += delta; + m_sharedMemory->getHeader()->used += delta; return offset; } @@ -227,11 +257,11 @@ void MonitoringData::write(ULONG offset, ULONG length, const void* buffer) // Write data item at the tail - UCHAR* const ptr = (UCHAR*) shared_memory->getHeader() + offset; + UCHAR* const ptr = (UCHAR*) m_sharedMemory->getHeader() + offset; Element* const element = (Element*) ptr; memcpy(ptr + sizeof(Element) + element->length, buffer, length); element->length += length; - shared_memory->getHeader()->used += length; + m_sharedMemory->getHeader()->used += length; } @@ -239,22 +269,22 @@ void MonitoringData::cleanup(AttNumber att_id) { // Remove information about the given session - for (ULONG offset = alignOffset(sizeof(Header)); offset < shared_memory->getHeader()->used;) + for (ULONG offset = alignOffset(sizeof(Header)); offset < m_sharedMemory->getHeader()->used;) { - UCHAR* const ptr = (UCHAR*) shared_memory->getHeader() + offset; + UCHAR* const ptr = (UCHAR*) m_sharedMemory->getHeader() + offset; const Element* const element = (Element*) ptr; const ULONG length = alignOffset(sizeof(Element) + element->length); if (element->attId == att_id) { - if (offset + length < shared_memory->getHeader()->used) + if (offset + length < m_sharedMemory->getHeader()->used) { - memmove(ptr, ptr + length, shared_memory->getHeader()->used - offset - length); - shared_memory->getHeader()->used -= length; + memmove(ptr, ptr + length, m_sharedMemory->getHeader()->used - offset - length); + m_sharedMemory->getHeader()->used -= length; } else { - shared_memory->getHeader()->used = offset; + m_sharedMemory->getHeader()->used = offset; } break; @@ -269,9 +299,9 @@ void MonitoringData::enumerate(SessionList& sessions, const char* user_name) { // Return IDs for all known (and permitted) sessions - for (ULONG offset = alignOffset(sizeof(Header)); offset < shared_memory->getHeader()->used;) + for (ULONG offset = alignOffset(sizeof(Header)); offset < m_sharedMemory->getHeader()->used;) { - UCHAR* const ptr = (UCHAR*) shared_memory->getHeader() + offset; + UCHAR* const ptr = (UCHAR*) m_sharedMemory->getHeader() + offset; const Element* const element = (Element*) ptr; const ULONG length = alignOffset(sizeof(Element) + element->length); @@ -285,19 +315,19 @@ void MonitoringData::enumerate(SessionList& sessions, const char* user_name) void MonitoringData::ensureSpace(ULONG length) { - ULONG newSize = shared_memory->getHeader()->used + length; + ULONG newSize = m_sharedMemory->getHeader()->used + length; - if (newSize > shared_memory->getHeader()->allocated) + if (newSize > m_sharedMemory->getHeader()->allocated) { newSize = FB_ALIGN(newSize, DEFAULT_SIZE); #ifdef HAVE_OBJECT_MAP FbLocalStatus statusVector; - if (!shared_memory->remapFile(&statusVector, newSize, true)) + if (!m_sharedMemory->remapFile(&statusVector, newSize, true)) { status_exception::raise(&statusVector); } - shared_memory->getHeader()->allocated = shared_memory->sh_mem_length_mapped; + m_sharedMemory->getHeader()->allocated = m_sharedMemory->sh_mem_length_mapped; #else status_exception::raise(Arg::Gds(isc_montabexh)); #endif @@ -317,7 +347,7 @@ bool MonitoringData::initialize(SharedMemoryBase* sm, bool initialize) { if (initialize) { - MonitoringHeader* header = reinterpret_cast(sm->sh_mem_header); + MonitoringHeader* const header = reinterpret_cast(sm->sh_mem_header); // Initialize the shared data header header->init(SharedMemoryBase::SRAM_DATABASE_SNAPSHOT, MONITOR_VERSION); diff --git a/src/jrd/Monitoring.h b/src/jrd/Monitoring.h index bfda2bae7d..d0dea9d9c7 100644 --- a/src/jrd/Monitoring.h +++ b/src/jrd/Monitoring.h @@ -248,14 +248,14 @@ private: }; -class MonitoringHeader : public Firebird::MemoryHeader +struct MonitoringHeader : public Firebird::MemoryHeader { -public: ULONG used; ULONG allocated; }; -class MonitoringData FB_FINAL : public Firebird::IpcObject + +class MonitoringData FB_FINAL : public Firebird::PermanentStorage, public Firebird::IpcObject { static const USHORT MONITOR_VERSION = 5; static const ULONG DEFAULT_SIZE = 1048576; @@ -333,6 +333,8 @@ public: bool initialize(Firebird::SharedMemoryBase*, bool); void mutexBug(int osErrorCode, const char* text); + void initSharedFile(); + void acquire(); void release(); @@ -350,7 +352,8 @@ private: void ensureSpace(ULONG); - Firebird::AutoPtr > shared_memory; + const Firebird::string m_dbId; + Firebird::AutoPtr > m_sharedMemory; }; diff --git a/src/jrd/event.cpp b/src/jrd/event.cpp index 63da78ab83..776760f107 100644 --- a/src/jrd/event.cpp +++ b/src/jrd/event.cpp @@ -132,10 +132,9 @@ EventManager::EventManager(const Firebird::string& id, Firebird::RefPtrremoveMapFile(); } release_shmem(); - - detach_shared_file(); } -void EventManager::attach_shared_file() +void EventManager::init_shared_file() { Firebird::PathName name; - get_shared_file_name(name); + name.printf(EVENT_FILE, m_dbId.c_str()); SharedMemory* tmp = FB_NEW_POOL(*getDefaultMemoryPool()) SharedMemory(name.c_str(), m_config->getEventMemSize(), this); @@ -193,18 +190,6 @@ void EventManager::attach_shared_file() } -void EventManager::detach_shared_file() -{ - delete m_sharedMemory.release(); -} - - -void EventManager::get_shared_file_name(Firebird::PathName& name) const -{ - name.printf(EVENT_FILE, m_dbId.c_str()); -} - - SLONG EventManager::create_session() { /************************************** @@ -528,26 +513,20 @@ void EventManager::acquire_shmem() while (SRQ_EMPTY(m_sharedMemory->getHeader()->evh_processes)) { - if (! m_sharedFileCreated) - { - // Someone is going to delete shared file? Reattach. - m_sharedMemory->mutexUnlock(); - detach_shared_file(); - - Thread::yield(); - - attach_shared_file(); - m_sharedMemory->mutexLock(); - } - else - { - // complete initialization - m_sharedFileCreated = false; - + if (m_sharedMemory->justCreated()) break; - } + + // Someone is going to delete shared file? Reattach. + m_sharedMemory->mutexUnlock(); + m_sharedMemory.reset(); + + Thread::yield(); + + init_shared_file(); + m_sharedMemory->mutexLock(); } - fb_assert(!m_sharedFileCreated); + + fb_assert(!m_sharedMemory->justCreated()); m_sharedMemory->getHeader()->evh_current_process = m_processOffset; @@ -1094,8 +1073,6 @@ bool EventManager::initialize(SharedMemoryBase* sm, bool init) * **************************************/ - m_sharedFileCreated = init; - // reset m_sharedMemory in advance to be able to use SRQ_BASE macro m_sharedMemory.reset(reinterpret_cast*>(sm)); diff --git a/src/jrd/event_proto.h b/src/jrd/event_proto.h index fa26259724..cb9fe0f918 100644 --- a/src/jrd/event_proto.h +++ b/src/jrd/event_proto.h @@ -88,9 +88,7 @@ private: void remove_que(srq*); bool request_completed(evt_req*); void watcher_thread(); - void attach_shared_file(); - void detach_shared_file(); - void get_shared_file_name(Firebird::PathName&) const; + void init_shared_file(); static void watcher_thread(EventManager* eventMgr) { diff --git a/src/jrd/replication/ChangeLog.cpp b/src/jrd/replication/ChangeLog.cpp index 6a098c9197..ec2277347f 100644 --- a/src/jrd/replication/ChangeLog.cpp +++ b/src/jrd/replication/ChangeLog.cpp @@ -66,9 +66,6 @@ using namespace Replication; namespace { - const char* SHMEM_FILE = "fb_repl_%s"; - const ULONG SHMEM_VERSION = 1; - const unsigned FLUSH_WAIT_INTERVAL = 1; // milliseconds const unsigned NO_SPACE_TIMEOUT = 10; // seconds @@ -316,15 +313,12 @@ ChangeLog::ChangeLog(MemoryPool& pool, const FB_UINT64 sequence, const Replication::Config* config) : PermanentStorage(pool), - m_database(pool, database), m_config(config), - m_segments(pool), m_sequence(sequence), m_shutdown(false) + m_dbId(pool, dbId), m_database(pool, database), + m_config(config), m_segments(pool), m_sequence(sequence), m_shutdown(false) { memcpy(&m_guid, &guid, sizeof(Guid)); - PathName filename; - filename.printf(SHMEM_FILE, dbId.c_str()); - - FB_NEW_POOL(pool) SharedMemory(filename.c_str(), STATE_MAPPING_SIZE, this); + initSharedFile(); { // scope LockGuard guard(this); @@ -332,7 +326,7 @@ ChangeLog::ChangeLog(MemoryPool& pool, // If the server crashes while archiving, segments may remain in the ARCH state forever. // This code allows to recover their state and retry archiving them. - const auto state = m_state->getHeader(); + const auto state = m_sharedMemory->getHeader(); if (!state->pidUpper) { @@ -373,39 +367,60 @@ ChangeLog::~ChangeLog() if (segment->getState() == SEGMENT_STATE_FULL) archiveSegment(segment); } + + m_sharedMemory->removeMapFile(); } } catch (const Exception&) {} // no-op clearSegments(); +} - if (m_state.hasData() && m_state->getHeader()) - delete m_state.release(); +void ChangeLog::initSharedFile() +{ + PathName filename; + filename.printf(REPL_FILE, m_dbId.c_str()); + + m_sharedMemory.reset(FB_NEW_POOL(getPool()) + SharedMemory(filename.c_str(), STATE_MAPPING_SIZE, this)); + + fb_assert(m_sharedMemory->getHeader()->mhb_type == SharedMemoryBase::SRAM_CHANGELOG_STATE); + fb_assert(m_sharedMemory->getHeader()->mhb_header_version == MemoryHeader::HEADER_VERSION); + fb_assert(m_sharedMemory->getHeader()->mhb_version == STATE_VERSION); } void ChangeLog::lockState() { - auto blockage = false; - - if (!m_state->mutexLockCond()) - { - blockage = true; - m_state->mutexLock(); - } + m_sharedMemory->mutexLock(); try { - const auto state = m_state->getHeader(); + while (!m_sharedMemory->getHeader()->pidUpper) + { + fb_assert(!m_sharedMemory->getHeader()->pidLower); - state->lockAcquires++; - if (blockage) - state->lockBlocks++; + if (m_sharedMemory->justCreated()) + break; + + // Someone is going to delete shared file? Reattach. + m_sharedMemory->mutexUnlock(); + m_sharedMemory.reset(); + + Thread::yield(); + + initSharedFile(); + m_sharedMemory->mutexLock(); + } + + fb_assert(!m_sharedMemory->justCreated()); + + const auto state = m_sharedMemory->getHeader(); if (m_segments.isEmpty() || state->segmentCount > m_segments.getCount()) initSegments(); } - catch (Exception&) + catch (const Exception&) { unlockState(); throw; @@ -414,14 +429,14 @@ void ChangeLog::lockState() void ChangeLog::unlockState() { - m_state->mutexUnlock(); + m_sharedMemory->mutexUnlock(); } void ChangeLog::linkSelf() { static const auto process_id = getpid(); - const auto state = m_state->getHeader(); + const auto state = m_sharedMemory->getHeader(); fb_assert(state->pidLower <= PID_CAPACITY); fb_assert(state->pidUpper <= PID_CAPACITY); @@ -471,7 +486,7 @@ bool ChangeLog::unlinkSelf() { static const auto process_id = getpid(); - const auto state = m_state->getHeader(); + const auto state = m_sharedMemory->getHeader(); fb_assert(state->pidLower <= PID_CAPACITY); fb_assert(state->pidUpper <= PID_CAPACITY); @@ -508,14 +523,12 @@ bool ChangeLog::unlinkSelf() bool ChangeLog::initialize(SharedMemoryBase* shmem, bool init) { - m_state.reset(reinterpret_cast*>(shmem)); - if (init) { - const auto state = m_state->getHeader(); + const auto state = reinterpret_cast(shmem->sh_mem_header); memset(state, 0, sizeof(State)); - state->init(SharedMemoryBase::SRAM_CHANGELOG_STATE, SHMEM_VERSION); + state->init(SharedMemoryBase::SRAM_CHANGELOG_STATE, STATE_VERSION); state->timestamp = time(NULL); state->sequence = m_sequence; @@ -563,7 +576,8 @@ FB_UINT64 ChangeLog::write(ULONG length, const UCHAR* data, bool sync) if (!segment) raiseError("Out of available space in changelog segments"); - const auto state = m_state->getHeader(); + const auto state = m_sharedMemory->getHeader(); + if (segment->getLength() == sizeof(SegmentHeader)) state->timestamp = time(NULL); @@ -740,7 +754,7 @@ void ChangeLog::switchActiveSegment() if (activeSegment) { - const auto state = m_state->getHeader(); + const auto state = m_sharedMemory->getHeader(); activeSegment->setState(SEGMENT_STATE_FULL); state->flushMark++; @@ -758,7 +772,7 @@ void ChangeLog::bgArchiver() { LockGuard guard(this); - const auto state = m_state->getHeader(); + const auto state = m_sharedMemory->getHeader(); for (const auto segment : m_segments) { @@ -804,7 +818,7 @@ void ChangeLog::bgArchiver() m_workingSemaphore.tryEnter(1); } } - catch (const Firebird::Exception& ex) + catch (const Exception& ex) { iscLogException("Error in changelog thread", ex); } @@ -815,7 +829,7 @@ void ChangeLog::bgArchiver() { m_cleanupSemaphore.release(); } - catch (const Firebird::Exception& ex) + catch (const Exception& ex) { iscLogException("Error while exiting changelog thread", ex); } @@ -825,7 +839,7 @@ void ChangeLog::initSegments() { clearSegments(); - const auto state = m_state->getHeader(); + const auto state = m_sharedMemory->getHeader(); for (auto iter = PathUtils::newDirIterator(getPool(), m_config->logDirectory); *iter; ++(*iter)) @@ -857,7 +871,7 @@ void ChangeLog::clearSegments() ChangeLog::Segment* ChangeLog::createSegment() { - const auto state = m_state->getHeader(); + const auto state = m_sharedMemory->getHeader(); const auto sequence = ++state->sequence; PathName filename; @@ -903,7 +917,7 @@ ChangeLog::Segment* ChangeLog::reuseSegment(ChangeLog::Segment* segment) // Rename the backing file - const auto state = m_state->getHeader(); + const auto state = m_sharedMemory->getHeader(); const auto sequence = ++state->sequence; PathName newname; @@ -956,7 +970,7 @@ ChangeLog::Segment* ChangeLog::getSegment(ULONG length) } } - const auto state = m_state->getHeader(); + const auto state = m_sharedMemory->getHeader(); if (activeSegment) { diff --git a/src/jrd/replication/ChangeLog.h b/src/jrd/replication/ChangeLog.h index 44fafe6ce5..c1797b5009 100644 --- a/src/jrd/replication/ChangeLog.h +++ b/src/jrd/replication/ChangeLog.h @@ -69,13 +69,18 @@ namespace Replication ULONG segmentCount; // number of segments in use ULONG flushMark; // last flush mark FB_UINT64 sequence; // sequence number of the last segment - FB_UINT64 lockAcquires; // number of state acquires - FB_UINT64 lockBlocks; // number of blocked state acquires ULONG pidLower; // Lower boundary mark in the PID array ULONG pidUpper; // Upper boundary mark in the PID array int pids[1]; // PIDs attached to the state }; + // Shared memory layout format + static const USHORT STATE_VERSION = 1; + // Mapping size (not extendable for the time being) + static const ULONG STATE_MAPPING_SIZE = 64 * 1024; // 64 KB + // Max number of processes accessing the shared state + static const ULONG PID_CAPACITY = (STATE_MAPPING_SIZE - offsetof(State, pids)) / sizeof(int); // ~16K + // RAII helper to lock the shared state class LockGuard @@ -184,11 +189,6 @@ namespace Replication #endif }; - // Mapping size (not extendable for the time being) - static const ULONG STATE_MAPPING_SIZE = 64 * 1024; // 64 KB - // Max number of processes accessing the shared state - static const ULONG PID_CAPACITY = (STATE_MAPPING_SIZE - offsetof(State, pids)) / sizeof(int); // ~16K - public: ChangeLog(Firebird::MemoryPool& pool, const Firebird::string& dbId, @@ -204,6 +204,8 @@ namespace Replication void bgArchiver(); private: + void initSharedFile(); + void lockState(); void unlockState(); @@ -230,10 +232,11 @@ namespace Replication void switchActiveSegment(); + const Firebird::string m_dbId; const Firebird::PathName m_database; const Config* const m_config; Firebird::Array m_segments; - Firebird::AutoPtr > m_state; + Firebird::AutoPtr > m_sharedMemory; Firebird::Guid m_guid; const FB_UINT64 m_sequence; diff --git a/src/lock/lock.cpp b/src/lock/lock.cpp index c5f78644e5..69ae324bdf 100644 --- a/src/lock/lock.cpp +++ b/src/lock/lock.cpp @@ -211,7 +211,6 @@ void LockManager::destroy(LockManager* lockMgr) LockManager::LockManager(const string& id, RefPtr conf) : PID(getpid()), m_bugcheck(false), - m_sharedFileCreated(false), m_process(NULL), m_processOffset(0), m_cleanupSync(getPool(), blocking_action_thread, THREAD_high), @@ -228,7 +227,7 @@ LockManager::LockManager(const string& id, RefPtr conf) { LocalStatus ls; CheckStatusWrapper localStatus(&ls); - if (!attach_shared_file(&localStatus)) + if (!init_shared_file(&localStatus)) { iscLogStatus("LockManager::LockManager()", &localStatus); status_exception::raise(&localStatus); @@ -293,7 +292,6 @@ LockManager::~LockManager() } } - detach_shared_file(&localStatus); #ifdef USE_SHMEM_EXT for (ULONG i = 1; i < m_extents.getCount(); ++i) { @@ -336,7 +334,7 @@ void* LockManager::ABS_PTR(SRQ_PTR item) #endif //USE_SHMEM_EXT -bool LockManager::attach_shared_file(CheckStatusWrapper* statusVector) +bool LockManager::init_shared_file(CheckStatusWrapper* statusVector) { PathName name; get_shared_file_name(name); @@ -365,22 +363,6 @@ bool LockManager::attach_shared_file(CheckStatusWrapper* statusVector) } -void LockManager::detach_shared_file(CheckStatusWrapper* statusVector) -{ - if (m_sharedMemory.hasData() && m_sharedMemory->getHeader()) - { - try - { - delete m_sharedMemory.release(); - } - catch (const Exception& ex) - { - ex.stuffException(statusVector); - } - } -} - - void LockManager::get_shared_file_name(PathName& name, ULONG extent) const { name.printf(LOCK_FILE, m_dbId.c_str()); @@ -1127,32 +1109,26 @@ void LockManager::acquire_shmem(SRQ_PTR owner_offset) while (SRQ_EMPTY(m_sharedMemory->getHeader()->lhb_processes)) { - if (!m_sharedFileCreated) + if (m_sharedMemory->justCreated()) { - // Someone is going to delete shared file? Reattach. - m_sharedMemory->mutexUnlock(); - detach_shared_file(&localStatus); - - Thread::yield(); - - if (!attach_shared_file(&localStatus)) - bug(NULL, "ISC_map_file failed (reattach shared file)"); - - m_sharedMemory->mutexLock(); - } - else - { - // complete initialization - m_sharedFileCreated = false; - // no sense thinking about statistics now m_blockage = false; - break; } + + // Someone is going to delete shared file? Reattach. + m_sharedMemory->mutexUnlock(); + m_sharedMemory.reset(); + + Thread::yield(); + + if (!init_shared_file(&localStatus)) + bug(NULL, "ISC_map_file failed (reattach shared file)"); + + m_sharedMemory->mutexLock(); } - fb_assert(!m_sharedFileCreated); + fb_assert(!m_sharedMemory->justCreated()); ++(m_sharedMemory->getHeader()->lhb_acquires); if (m_blockage) @@ -2324,8 +2300,6 @@ bool LockManager::initialize(SharedMemoryBase* sm, bool initializeMemory) * **************************************/ - m_sharedFileCreated = initializeMemory; - // reset m_sharedMemory in advance to be able to use SRQ_BASE macro m_sharedMemory.reset(reinterpret_cast*>(sm)); @@ -2339,9 +2313,7 @@ bool LockManager::initialize(SharedMemoryBase* sm, bool initializeMemory) #endif if (!initializeMemory) - { return true; - } lhb* hdr = m_sharedMemory->getHeader(); memset(hdr, 0, sizeof(lhb)); diff --git a/src/lock/lock_proto.h b/src/lock/lock_proto.h index 3d38dd0176..1482d49d7d 100644 --- a/src/lock/lock_proto.h +++ b/src/lock/lock_proto.h @@ -470,8 +470,7 @@ private: void validate_shb(const SRQ_PTR); void wait_for_request(thread_db*, lrq*, SSHORT); - bool attach_shared_file(Firebird::CheckStatusWrapper*); - void detach_shared_file(Firebird::CheckStatusWrapper*); + bool init_shared_file(Firebird::CheckStatusWrapper*); void get_shared_file_name(Firebird::PathName&, ULONG extend = 0) const; static void blocking_action_thread(LockManager* lockMgr) @@ -483,7 +482,6 @@ private: void mutexBug(int osErrorCode, const char* text); bool m_bugcheck; - bool m_sharedFileCreated; prc* m_process; SRQ_PTR m_processOffset; From de57ae7f5e2d885da9808f7e160467fbae529e1e Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sun, 1 Dec 2019 10:32:06 -0300 Subject: [PATCH 123/274] Fixes for MacOS build. --- builds/posix/make.shared.targets | 4 ++-- builds/posix/prefix.darwin_x86_64 | 2 +- configure.ac | 1 + src/common/os/darwin/mod_loader.cpp | 10 ++++++---- src/common/os/os_utils.h | 7 +++++++ src/gpre/c_cxx.cpp | 4 ---- src/gpre/obj_cxx.cpp | 4 ---- 7 files changed, 17 insertions(+), 15 deletions(-) diff --git a/builds/posix/make.shared.targets b/builds/posix/make.shared.targets index 7f3682baba..ae5ee5951d 100644 --- a/builds/posix/make.shared.targets +++ b/builds/posix/make.shared.targets @@ -48,8 +48,8 @@ $(OBJ)/dsql/parse.cpp $(SRC_ROOT)/include/gen/parse.h: $(SRC_ROOT)/dsql/parse.y ($(BTYACC) -l -d -S $(SRC_ROOT)/dsql/btyacc_fb.ske $(GEN_ROOT)/y.y; echo $$? > $(GEN_ROOT)/y.status) 2>&1 | tee $(GEN_ROOT)/y.txt (exit `cat $(GEN_ROOT)/y.status`) sed -n -e "s/.*btyacc: \(.*conflicts.*\)/\1/p" $(GEN_ROOT)/y.txt > $(SRC_ROOT)/dsql/parse-conflicts.txt - sed -i 's/#define \([A-Z].*\)/#define TOK_\1/' $(GEN_ROOT)/y_tab.h - sed -i 's/#define TOK_YY\(.*\)/#define YY\1/' $(GEN_ROOT)/y_tab.h + sed -i -e 's/#define \([A-Z].*\)/#define TOK_\1/' $(GEN_ROOT)/y_tab.h + sed -i -e 's/#define TOK_YY\(.*\)/#define YY\1/' $(GEN_ROOT)/y_tab.h $(MV) $(GEN_ROOT)/y_tab.h $(SRC_ROOT)/include/gen/parse.h $(MV) $(GEN_ROOT)/y_tab.c $(OBJ)/dsql/parse.cpp touch $(OBJ)/dsql/parse.cpp diff --git a/builds/posix/prefix.darwin_x86_64 b/builds/posix/prefix.darwin_x86_64 index c846a08ec7..04d832733f 100644 --- a/builds/posix/prefix.darwin_x86_64 +++ b/builds/posix/prefix.darwin_x86_64 @@ -28,7 +28,7 @@ export DYLD_LIBRARY_PATH DYLD_FALLBACK_LIBRARY_PATH=/opt/local/lib export DYLD_FALLBACK_LIBRARY_PATH -MACOSX_DEPLOYMENT_TARGET=10.7 +MACOSX_DEPLOYMENT_TARGET=10.9 export MACOSX_DEPLOYMENT_TARGET PROD_FLAGS=-O1 -DDARWIN -pipe -MMD -fPIC -fno-common -mmacosx-version-min=10.7 diff --git a/configure.ac b/configure.ac index 00d1070344..ac5a445f82 100644 --- a/configure.ac +++ b/configure.ac @@ -97,6 +97,7 @@ case "$build" in SHRLIB_EXT=dylib CPU_TYPE=x86_64 EXPORT_SYMBOLS_STYLE=darwin + RAW_DEVICES_FLG=N ;; i*86-*-darwin*) diff --git a/src/common/os/darwin/mod_loader.cpp b/src/common/os/darwin/mod_loader.cpp index 70f24b3507..8c0eae6d18 100644 --- a/src/common/os/darwin/mod_loader.cpp +++ b/src/common/os/darwin/mod_loader.cpp @@ -28,6 +28,7 @@ #include "firebird.h" #include "../common/os/mod_loader.h" +#include "../common/os/os_utils.h" #include "../../common.h" #include #include @@ -42,8 +43,9 @@ class DlfcnModule : public ModuleLoader::Module { public: - DlfcnModule(void* m) - : module(m) + DlfcnModule(MemoryPool& pool, const Firebird::PathName& aFileName, void* m) + : ModuleLoader::Module(pool, aFileName), + module(m) {} ~DlfcnModule(); @@ -55,7 +57,7 @@ private: bool ModuleLoader::isLoadableModule(const Firebird::PathName& module) { - struct STAT sb; + struct stat sb; if (-1 == os_utils::stat(module.c_str(), &sb)) return false; @@ -123,7 +125,7 @@ ModuleLoader::Module* ModuleLoader::loadModule(ISC_STATUS* status, const Firebir return 0; } - return FB_NEW_POOL(*getDefaultMemoryPool()) DlfcnModule(module); + return FB_NEW_POOL(*getDefaultMemoryPool()) DlfcnModule(*getDefaultMemoryPool(), modPath, module); } DlfcnModule::~DlfcnModule() diff --git a/src/common/os/os_utils.h b/src/common/os/os_utils.h index c4c70857d0..45c474552d 100644 --- a/src/common/os/os_utils.h +++ b/src/common/os/os_utils.h @@ -292,6 +292,7 @@ namespace os_utils return rc; } +#ifdef HAVE_POSIX_FADVISE inline int posix_fadvise(int fd, off_t offset, off_t len, int advice) { int rc; @@ -307,6 +308,12 @@ namespace os_utils return rc; } +#else + inline int posix_fadvise(int, off_t, off_t, int) + { + return 0; + } +#endif inline int getrlimit(int resource, struct rlimit* rlim) { diff --git a/src/gpre/c_cxx.cpp b/src/gpre/c_cxx.cpp index 6e879484d8..09da5a5471 100644 --- a/src/gpre/c_cxx.cpp +++ b/src/gpre/c_cxx.cpp @@ -138,11 +138,7 @@ static const char* const NULL_STRING = "(char*) 0"; static const char* const NULL_STATUS = "NULL"; static const char* const NULL_SQLDA = "NULL"; -#ifdef DARWIN -static const char* const GDS_INCLUDE = ""; -#else static const char* const GDS_INCLUDE = ""; -#endif static const char* const DCL_LONG = "ISC_LONG"; static const char* const DCL_QUAD = "ISC_QUAD"; diff --git a/src/gpre/obj_cxx.cpp b/src/gpre/obj_cxx.cpp index ecb6ea316c..6a41b7d449 100644 --- a/src/gpre/obj_cxx.cpp +++ b/src/gpre/obj_cxx.cpp @@ -138,11 +138,7 @@ static const char* const NULL_STRING = "NULL"; static const char* const NULL_STATUS = "NULL"; static const char* const NULL_SQLDA = "NULL"; -#ifdef DARWIN -static const char* const GDS_INCLUDE = ""; -#else static const char* const GDS_INCLUDE = ""; -#endif static const char* const DCL_LONG = "ISC_LONG"; static const char* const DCL_QUAD = "ISC_QUAD"; From 0ca45880d3f85c85299349e8b2dd3cc69d494e0a Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Mon, 2 Dec 2019 00:03:56 +0000 Subject: [PATCH 124/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index e6615f4206..fb58c71fd1 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1655 + FORMAL BUILD NUMBER:1658 */ -#define PRODUCT_VER_STRING "4.0.0.1655" -#define FILE_VER_STRING "WI-T4.0.0.1655" -#define LICENSE_VER_STRING "WI-T4.0.0.1655" -#define FILE_VER_NUMBER 4, 0, 0, 1655 +#define PRODUCT_VER_STRING "4.0.0.1658" +#define FILE_VER_STRING "WI-T4.0.0.1658" +#define LICENSE_VER_STRING "WI-T4.0.0.1658" +#define FILE_VER_NUMBER 4, 0, 0, 1658 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1655" +#define FB_BUILD_NO "1658" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 24deae3b5f..db776331cc 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1655 +BuildNum=1658 NowAt=`pwd` cd `dirname $0` From 6c1744a55c4c04477249b7b6fd28f60b21d6063c Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Wed, 27 Nov 2019 12:19:12 -0300 Subject: [PATCH 125/274] Setup GitHub Actions for Linux and Windows. --- .github/workflows/main.yml | 151 +++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000000..329fa6bd0a --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,151 @@ +name: CI + +on: [push, pull_request] + +jobs: + build: + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + # Build is segfaulting in Ubuntu. + #os: [macOS-latest, windows-2016, ubuntu-16.04] + os: [windows-2016] + platform: [x64, x86] + #exclude: + # - os: macOS-latest + # platform: x86 + # - os: ubuntu-16.04 + # platform: x86 + + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 10 + + - name: Prepare (Linux) + if: matrix.os == 'ubuntu-16.04' + run: | + sudo apt-get install libtool-bin libtommath0 libtommath-dev libicu-dev zlib1g-dev + + - name: Build (Linux) + if: matrix.os == 'ubuntu-16.04' + run: | + CC=clang CXX=clang++ ./autogen.sh --enable-binreloc --with-builtin-tomcrypt --prefix=/opt/firebird + make + make dist + tar xzvf gen/Firebird-[0-9]*.tar.gz + (cd Firebird-[0-9]*; sudo ./install.sh -silent) + + - name: Prepare (MacOS) + if: matrix.os == 'macOS-latest' + run: | + brew install automake libtool + export LIBTOOLIZE=glibtoolize + export LIBTOOL=glibtool + + mkdir extern/icu-macos + pushd extern/icu-macos + wget https://github.com/unicode-org/icu/releases/download/release-59-2/icu4c-59_2-src.tgz + wget https://github.com/unicode-org/icu/commit/24aeb9a5a5874f4ce5db912e30670ac3ae236971.patch + tar xzvf icu4c-59_2-src.tgz + ICU_INSTALL_PATH=`pwd`/install + cd icu/source + patch -p3 < ../../24aeb9a5a5874f4ce5db912e30670ac3ae236971.patch + ./runConfigureICU MacOSX --prefix=$ICU_INSTALL_PATH + make -j4 + make install + install_name_tool -id @rpath/lib/libicuuc.dylib $ICU_INSTALL_PATH/lib/libicuuc.dylib + install_name_tool -id @rpath/lib/libicui18n.dylib $ICU_INSTALL_PATH/lib/libicui18n.dylib + install_name_tool -id @rpath/lib/libicudata.dylib $ICU_INSTALL_PATH/lib/libicudata.dylib + install_name_tool -change libicudata.59.dylib @loader_path/libicudata.59.dylib $ICU_INSTALL_PATH/lib/libicuuc.59.dylib + install_name_tool -change libicudata.59.dylib @loader_path/libicudata.59.dylib $ICU_INSTALL_PATH/lib/libicui18n.59.dylib + install_name_tool -change libicuuc.59.dylib @loader_path/libicuuc.59.dylib $ICU_INSTALL_PATH/lib/libicui18n.59.dylib + popd + mkdir -p gen/Release/firebird/lib + mkdir -p gen/Debug/firebird/lib + cp $ICU_INSTALL_PATH/lib/*.dylib gen/Release/firebird/lib/ + cp $ICU_INSTALL_PATH/lib/*.dylib gen/Debug/firebird/lib/ + + - name: Build (MacOS) + if: matrix.os == 'macOS-latest' + run: | + export LIBTOOLIZE=glibtoolize + export LIBTOOL=glibtool + + ICU_INSTALL_PATH=`pwd`/extern/icu-macos/install + + export C_INCLUDE_PATH="$ICU_INSTALL_PATH/include:$C_INCLUDE_PATH" + export CPLUS_INCLUDE_PATH="$ICU_INSTALL_PATH/include:$CPLUS_INCLUDE_PATH" + + LIBRARY_PATH="$ICU_INSTALL_PATH/lib:$LIBRARY_PATH" ./autogen.sh --with-builtin-tommath + make -j4 + + (cd gen; make -B -f make.platform.postfix ICU_LOC="$ICU_INSTALL_PATH/lib/") + (cd gen; make -B -f Makefile.install) + + # Rename directory to make sure the build is relocatable. + mv gen gen2 + sudo installer -pkg gen2/Release/*.pkg -verbose -target / + + export FIREBIRD_LOCK=`pwd`/temp + echo "create database 't.fdb'; select '1' from rdb\$database; select _win1252 '2' from rdb\$database; select _utf8 '3' collate unicode from rdb\$database;" | /Library/Frameworks/Firebird.framework/Resources/bin/isql + + echo "create database 'localhost:/tmp/t.fdb' user sysdba password 'masterkey'; select '11' from rdb\$database; select _win1252 '22' from rdb\$database; select _utf8 '33' collate unicode from rdb\$database;" | /Library/Frameworks/Firebird.framework/Resources/bin/isql + + mv gen2 gen + mkdir gen/artifacts + mv gen/Release/*.pkg gen/artifacts + + - name: Prepare (Windows) + if: matrix.os == 'windows-2016' + shell: cmd + run: | + for /r %%i in (*.bat) do unix2dos "%%i" + + - name: Build (Windows) + if: matrix.os == 'windows-2016' + shell: cmd + env: + PLATFORM: ${{ matrix.platform }} + run: | + set + if "%PLATFORM%" == "x64" set FB_VS_ARCH=amd64 + if "%PLATFORM%" == "x64" set FB_PROCESSOR_ARCHITECTURE=AMD64 + if "%PLATFORM%" == "x64" set FB_OUTPUT_SUFFIX=x64 + if "%PLATFORM%" == "x86" set FB_VS_ARCH=x86 + if "%PLATFORM%" == "x86" set FB_PROCESSOR_ARCHITECTURE=x86 + if "%PLATFORM%" == "x86" set FB_OUTPUT_SUFFIX=win32 + set + call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=%FB_VS_ARCH% + cd builds\win32 + run_all.bat JUSTBUILD + + - name: Upload (Linux) + if: matrix.os == 'ubuntu-16.04' + uses: actions/upload-artifact@master + with: + name: firebird-linux + path: gen/Firebird-[0-9]*.tar.gz + + - name: Upload (MacOS) + if: matrix.os == 'macOS-latest' + uses: actions/upload-artifact@master + with: + name: firebird-macos + path: gen/artifacts + + - name: Upload (Windows x64) + if: matrix.os == 'windows-2016' && matrix.platform == 'x64' + uses: actions/upload-artifact@master + with: + name: firebird-windows-x64 + path: output_x64 + + - name: Upload (Windows x86) + if: matrix.os == 'windows-2016' && matrix.platform == 'x86' + uses: actions/upload-artifact@master + with: + name: firebird-windows-x86 + path: output_win32 From 2e8060f08143cc512311fc30b7f2d7d3b964e6f4 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Thu, 28 Nov 2019 12:54:50 -0300 Subject: [PATCH 126/274] Frontport changes for MacOS build. --- .github/workflows/main.yml | 26 +-- .travis.yml | 88 ++++++-- .../arch-specific/darwin/Distribution.xml | 19 ++ .../install/arch-specific/darwin/Makefile.in | 129 +++-------- .../install/arch-specific/darwin/Readme.txt | 11 +- .../install/arch-specific/darwin/Welcome.txt | 4 +- .../arch-specific/darwin/changeServerMode | 134 +++++++++++ .../install/arch-specific/darwin/embed.darwin | 109 +++++---- .../arch-specific/darwin/install-script | 84 ++++--- .../darwin/launchd.org.firebird.gds.plist | 27 +-- .../darwin/launchdcs.org.firebird.gds.plist | 35 +++ .../arch-specific/darwin/preupgrade-script | 16 +- builds/posix/Makefile.in | 69 +++--- builds/posix/darwin.defaults | 15 +- builds/posix/make.defaults | 48 +++- builds/posix/postfix.darwin | 209 +++++------------- builds/posix/prefix.darwin_x86_64 | 6 +- extern/cloop/Makefile | 2 + extern/libtommath/makefile.shared | 24 +- src/common/unicode_util.cpp | 8 +- 20 files changed, 586 insertions(+), 477 deletions(-) create mode 100644 builds/install/arch-specific/darwin/Distribution.xml create mode 100755 builds/install/arch-specific/darwin/changeServerMode create mode 100644 builds/install/arch-specific/darwin/launchdcs.org.firebird.gds.plist diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 329fa6bd0a..3d7d1490fe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,11 +11,11 @@ jobs: matrix: # Build is segfaulting in Ubuntu. #os: [macOS-latest, windows-2016, ubuntu-16.04] - os: [windows-2016] + os: [macOS-latest, windows-2016] platform: [x64, x86] - #exclude: - # - os: macOS-latest - # platform: x86 + exclude: + - os: macOS-latest + platform: x86 # - os: ubuntu-16.04 # platform: x86 @@ -47,9 +47,9 @@ jobs: mkdir extern/icu-macos pushd extern/icu-macos - wget https://github.com/unicode-org/icu/releases/download/release-59-2/icu4c-59_2-src.tgz - wget https://github.com/unicode-org/icu/commit/24aeb9a5a5874f4ce5db912e30670ac3ae236971.patch - tar xzvf icu4c-59_2-src.tgz + curl -OL https://github.com/unicode-org/icu/releases/download/release-63-2/icu4c-63_2-src.tgz + curl -OL https://github.com/unicode-org/icu/commit/24aeb9a5a5874f4ce5db912e30670ac3ae236971.patch + tar xzf icu4c-63_2-src.tgz ICU_INSTALL_PATH=`pwd`/install cd icu/source patch -p3 < ../../24aeb9a5a5874f4ce5db912e30670ac3ae236971.patch @@ -59,14 +59,14 @@ jobs: install_name_tool -id @rpath/lib/libicuuc.dylib $ICU_INSTALL_PATH/lib/libicuuc.dylib install_name_tool -id @rpath/lib/libicui18n.dylib $ICU_INSTALL_PATH/lib/libicui18n.dylib install_name_tool -id @rpath/lib/libicudata.dylib $ICU_INSTALL_PATH/lib/libicudata.dylib - install_name_tool -change libicudata.59.dylib @loader_path/libicudata.59.dylib $ICU_INSTALL_PATH/lib/libicuuc.59.dylib - install_name_tool -change libicudata.59.dylib @loader_path/libicudata.59.dylib $ICU_INSTALL_PATH/lib/libicui18n.59.dylib - install_name_tool -change libicuuc.59.dylib @loader_path/libicuuc.59.dylib $ICU_INSTALL_PATH/lib/libicui18n.59.dylib + install_name_tool -change libicudata.63.dylib @loader_path/libicudata.63.dylib $ICU_INSTALL_PATH/lib/libicuuc.63.dylib + install_name_tool -change libicudata.63.dylib @loader_path/libicudata.63.dylib $ICU_INSTALL_PATH/lib/libicui18n.63.dylib + install_name_tool -change libicuuc.63.dylib @loader_path/libicuuc.63.dylib $ICU_INSTALL_PATH/lib/libicui18n.63.dylib popd mkdir -p gen/Release/firebird/lib mkdir -p gen/Debug/firebird/lib - cp $ICU_INSTALL_PATH/lib/*.dylib gen/Release/firebird/lib/ - cp $ICU_INSTALL_PATH/lib/*.dylib gen/Debug/firebird/lib/ + cp $ICU_INSTALL_PATH/lib/libicu{data,i18n,uc}.*dylib gen/Release/firebird/lib/ + cp $ICU_INSTALL_PATH/lib/libicu{data,i18n,uc}.*dylib gen/Debug/firebird/lib/ - name: Build (MacOS) if: matrix.os == 'macOS-latest' @@ -79,7 +79,7 @@ jobs: export C_INCLUDE_PATH="$ICU_INSTALL_PATH/include:$C_INCLUDE_PATH" export CPLUS_INCLUDE_PATH="$ICU_INSTALL_PATH/include:$CPLUS_INCLUDE_PATH" - LIBRARY_PATH="$ICU_INSTALL_PATH/lib:$LIBRARY_PATH" ./autogen.sh --with-builtin-tommath + LIBRARY_PATH="$ICU_INSTALL_PATH/lib:$LIBRARY_PATH" ./autogen.sh --with-builtin-tommath --with-builtin-tomcrypt make -j4 (cd gen; make -B -f make.platform.postfix ICU_LOC="$ICU_INSTALL_PATH/lib/") diff --git a/.travis.yml b/.travis.yml index d5b8f0a348..43e99f47d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,11 @@ +matrix: + include: + - os: osx + osx_image: xcode9.2 # macOS 10.12 Sierra + - os: osx + osx_image: xcode11.2 # macOS 10.14 Mojave + - os: linux + language: cpp notifications: @@ -6,25 +14,65 @@ notifications: sudo: required dist: xenial -branches: - except: - - B2_5_Release - - B2_1_Release - - B2_0_Release - - B1_5_Release - -addons: - apt: - packages: - - libtool-bin - - libtommath0 - - libtommath-dev - - libicu-dev - - zlib1g-dev +install: + - | + if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + sudo apt-get update + sudo apt-get install -y libtool-bin libtommath0 libtommath-dev libicu-dev zlib1g-dev + fi script: - - ./autogen.sh --enable-binreloc --with-builtin-tomcrypt --prefix=/opt/firebird - - make -j4 - - make dist - - tar xzvf gen/Firebird-[0-9]*.tar.gz - - (cd Firebird-[0-9]*; sudo ./install.sh -silent) + - | + if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + ./autogen.sh --enable-binreloc --with-builtin-tomcrypt --prefix=/opt/firebird + make -j4 + make dist + tar xzvf gen/Firebird-[0-9]*.tar.gz + (cd Firebird-[0-9]*; sudo ./install.sh -silent) + fi + + if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + export LIBTOOLIZE=glibtoolize + export LIBTOOL=glibtool + + mkdir extern/icu-macos + pushd extern/icu-macos + curl -OL https://github.com/unicode-org/icu/releases/download/release-63-2/icu4c-63_2-src.tgz + curl -OL https://github.com/unicode-org/icu/commit/24aeb9a5a5874f4ce5db912e30670ac3ae236971.patch + tar xzf icu4c-63_2-src.tgz + ICU_INSTALL_PATH=`pwd`/install + cd icu/source + patch -p3 < ../../24aeb9a5a5874f4ce5db912e30670ac3ae236971.patch + ./runConfigureICU MacOSX --prefix=$ICU_INSTALL_PATH + make -j4 + make install + install_name_tool -id @rpath/lib/libicuuc.dylib $ICU_INSTALL_PATH/lib/libicuuc.dylib + install_name_tool -id @rpath/lib/libicui18n.dylib $ICU_INSTALL_PATH/lib/libicui18n.dylib + install_name_tool -id @rpath/lib/libicudata.dylib $ICU_INSTALL_PATH/lib/libicudata.dylib + install_name_tool -change libicudata.63.dylib @loader_path/libicudata.63.dylib $ICU_INSTALL_PATH/lib/libicuuc.63.dylib + install_name_tool -change libicudata.63.dylib @loader_path/libicudata.63.dylib $ICU_INSTALL_PATH/lib/libicui18n.63.dylib + install_name_tool -change libicuuc.63.dylib @loader_path/libicuuc.63.dylib $ICU_INSTALL_PATH/lib/libicui18n.63.dylib + popd + mkdir -p gen/Release/firebird/lib + mkdir -p gen/Debug/firebird/lib + cp $ICU_INSTALL_PATH/lib/libicu{data,i18n,uc}.*dylib gen/Release/firebird/lib/ + cp $ICU_INSTALL_PATH/lib/libicu{data,i18n,uc}.*dylib gen/Debug/firebird/lib/ + + export C_INCLUDE_PATH="$ICU_INSTALL_PATH/include:$C_INCLUDE_PATH" + export CPLUS_INCLUDE_PATH="$ICU_INSTALL_PATH/include:$CPLUS_INCLUDE_PATH" + + LIBRARY_PATH="$ICU_INSTALL_PATH/lib:$LIBRARY_PATH" ./autogen.sh --with-builtin-tommath --with-builtin-tomcrypt + make -j4 + + (cd gen; make -B -f make.platform.postfix ICU_LOC="$ICU_INSTALL_PATH/lib/") + (cd gen; make -B -f Makefile.install) + + # Rename directory to make sure the build is relocatable. + mv gen gen2 + sudo installer -pkg gen2/Release/*.pkg -verbose -target / + + export FIREBIRD_LOCK=`pwd`/temp + echo "create database 't.fdb'; select '1' from rdb\$database; select _win1252 '2' from rdb\$database; select _utf8 '3' collate unicode from rdb\$database;" | /Library/Frameworks/Firebird.framework/Resources/bin/isql + + echo "create database 'localhost:/tmp/t.fdb' user sysdba password 'masterkey'; select '11' from rdb\$database; select _win1252 '22' from rdb\$database; select _utf8 '33' collate unicode from rdb\$database;" | /Library/Frameworks/Firebird.framework/Resources/bin/isql + fi diff --git a/builds/install/arch-specific/darwin/Distribution.xml b/builds/install/arch-specific/darwin/Distribution.xml new file mode 100644 index 0000000000..e5471e3686 --- /dev/null +++ b/builds/install/arch-specific/darwin/Distribution.xml @@ -0,0 +1,19 @@ + + + Firebird Database Server + + + + + + + + + + + + + + + Firebird.pkg + diff --git a/builds/install/arch-specific/darwin/Makefile.in b/builds/install/arch-specific/darwin/Makefile.in index 342d4da5a7..41cd0c2eda 100644 --- a/builds/install/arch-specific/darwin/Makefile.in +++ b/builds/install/arch-specific/darwin/Makefile.in @@ -1,3 +1,5 @@ +TARGET ?= Release + # EKU: taken from Makefile.in.firebird ROOT=.. @@ -18,10 +20,10 @@ FB_REV_NO:=$(shell cpp -DDARWIN -I. ../builds/install/arch-specific/darwin/revno FB_BUILD_NO:=$(shell cpp -DDARWIN -I. ../builds/install/arch-specific/darwin/buildno.c | tail -2 | sed -e 's/" "//g' -e 's/"//g') FB_PLATFORM:=$(CpuType) -package packages dist: package_@FIREBIRD_ARCH_TYPE@ +package: package_firebird -package_classic package_embedded: INST_NM=FirebirdCS-$(FB_MAJOR_VERS).$(FB_MINOR_VERS).$(FB_REV_NO)-$(FB_BUILD_NO)-$(FB_PLATFORM) -package_classic package_embedded: +package_firebird: INST_NM=Firebird-$(FB_MAJOR_VERS).$(FB_MINOR_VERS).$(FB_REV_NO)-$(FB_BUILD_NO)-$(FB_PLATFORM) +package_firebird: sed -e 's/_FB_BUILD_SUFFIX_/$(FB_VER_SUFFIX)/g' \ -e 's/_SMFB_BUILD_SUFFIX_/$(FB_VER_SUFFIX_SM)/g' \ -e 's/_MFB_BUILD_SUFFIX_/$(FB_VER_SUFFIX_M)/g' \ @@ -29,7 +31,7 @@ package_classic package_embedded: -e 's/_MINOR_VERS_/$(FB_MINOR_VERS)/g' \ -e 's/_REV_NO_/$(FB_REV_NO)/g' \ ../builds/install/arch-specific/darwin/Info.plist \ - > $(ROOT)/gen/firebird/frameworks/FirebirdCS.framework/Resources/Info.plist + > $(ROOT)/gen/$(TARGET)/frameworks/Firebird4.framework/Resources/Info.plist sed -e 's/_FB_BUILD_SUFFIX_/$(FB_VER_SUFFIX)/g' \ -e 's/_SMFB_BUILD_SUFFIX_/$(FB_VER_SUFFIX_SM)/g' \ -e 's/_MFB_BUILD_SUFFIX_/$(FB_VER_SUFFIX_M)/g' \ @@ -37,99 +39,40 @@ package_classic package_embedded: -e 's/_MINOR_VERS_/$(FB_MINOR_VERS)/g' \ -e 's/_REV_NO_/$(FB_REV_NO)/g' \ ../builds/install/arch-specific/darwin/Description.plist \ - > $(ROOT)/gen/firebird/frameworks/FirebirdCS.framework/Resources/Description.plist - - rm -fr $(ROOT)/gen/firebird/scripts - mkdir $(ROOT)/gen/firebird/scripts - cp ../builds/install/arch-specific/darwin/install-script \ - $(ROOT)/gen/firebird/scripts/postinstall - cp ../builds/install/arch-specific/darwin/preupgrade-script \ - $(ROOT)/gen/firebird/scripts/preinstall - chmod u+x $(ROOT)/gen/firebird/scripts/postinstall - chmod u+x $(ROOT)/gen/firebird/scripts/preinstall - - rm -fr $(ROOT)/gen/firebird/resources - mkdir $(ROOT)/gen/firebird/resources - cp ../builds/install/arch-specific/darwin/Welcome.txt \ - $(ROOT)/gen/firebird/resources/Welcome.txt - cp ../builds/install/arch-specific/darwin/License.txt \ - $(ROOT)/gen/firebird/resources/License.txt - cp ../builds/install/arch-specific/darwin/DistributionCS.xml \ - $(ROOT)/gen/firebird/DistributionCS.xml + > $(ROOT)/gen/$(TARGET)/frameworks/Firebird4.framework/Resources/Description.plist - rm -fr firebird/packages - mkdir firebird/packages - pkgbuild --root $(ROOT)/gen/firebird/frameworks/FirebirdCS.framework \ + rm -fr $(ROOT)/gen/$(TARGET)/scripts + mkdir $(ROOT)/gen/$(TARGET)/scripts + cp ../builds/install/arch-specific/darwin/install-script \ + $(ROOT)/gen/$(TARGET)/scripts/postinstall + cp ../builds/install/arch-specific/darwin/preupgrade-script \ + $(ROOT)/gen/$(TARGET)/scripts/preinstall + chmod u+x $(ROOT)/gen/$(TARGET)/scripts/postinstall + chmod u+x $(ROOT)/gen/$(TARGET)/scripts/preinstall + + rm -fr $(ROOT)/gen/$(TARGET)/resources + mkdir $(ROOT)/gen/$(TARGET)/resources + cp ../builds/install/arch-specific/darwin/Welcome.txt \ + $(ROOT)/gen/$(TARGET)/resources/Welcome.txt + cp ../builds/install/arch-specific/darwin/Readme.txt \ + $(ROOT)/gen/$(TARGET)/resources/Readme.txt + cp ../builds/install/arch-specific/darwin/License.txt \ + $(ROOT)/gen/$(TARGET)/resources/License.txt + cp ../builds/install/arch-specific/darwin/Distribution.xml \ + $(ROOT)/gen/$(TARGET)/Distribution.xml + + rm -fr $(TARGET)/packages + mkdir $(TARGET)/packages + pkgbuild --root $(ROOT)/gen/$(TARGET)/frameworks/Firebird4.framework \ --identifier com.firebirdsql.Firebird \ --install-location /Library/Frameworks/Firebird.framework \ - --scripts $(ROOT)/gen/firebird/scripts \ - firebird/packages/FirebirdCS.pkg + --scripts $(ROOT)/gen/$(TARGET)/scripts \ + $(TARGET)/packages/Firebird.pkg - productbuild --distribution firebird/DistributionCS.xml \ - --resources firebird/resources \ - --package-path firebird/packages \ - firebird/$(INST_NM).pkg + productbuild --distribution $(TARGET)/Distribution.xml \ + --resources $(TARGET)/resources \ + --package-path $(TARGET)/packages \ + $(TARGET)/$(INST_NM).pkg -package_super package_server: INST_NM=FirebirdSS-$(FB_MAJOR_VERS).$(FB_MINOR_VERS).$(FB_REV_NO)-$(FB_BUILD_NO)-$(FB_PLATFORM) -package_super package_server: - sed -e 's/_FB_BUILD_SUFFIX_/$(FB_VER_SUFFIX)/g' \ - -e 's/_SMFB_BUILD_SUFFIX_/$(FB_VER_SUFFIX_SM)/g' \ - -e 's/_MFB_BUILD_SUFFIX_/$(FB_VER_SUFFIX_M)/g' \ - -e 's/_MAJOR_VERS_/$(FB_MAJOR_VERS)/g' \ - -e 's/_MINOR_VERS_/$(FB_MINOR_VERS)/g' \ - -e 's/_REV_NO_/$(FB_REV_NO)/g' \ - ../builds/install/arch-specific/darwin/Info.plist \ - > $(ROOT)/gen/firebird/frameworks/FirebirdSS.framework/Resources/Info.plist - sed -e 's/_FB_BUILD_SUFFIX_/$(FB_VER_SUFFIX)/g' \ - -e 's/_SMFB_BUILD_SUFFIX_/$(FB_VER_SUFFIX_SM)/g' \ - -e 's/_MFB_BUILD_SUFFIX_/$(FB_VER_SUFFIX_M)/g' \ - -e 's/_MAJOR_VERS_/$(FB_MAJOR_VERS)/g' \ - -e 's/_MINOR_VERS_/$(FB_MINOR_VERS)/g' \ - -e 's/_REV_NO_/$(FB_REV_NO)/g' \ - ../builds/install/arch-specific/darwin/Description.plist \ - > $(ROOT)/gen/firebird/frameworks/FirebirdSS.framework/Resources/Description.plist - rm -fr $(ROOT)/gen/firebird/scripts - mkdir $(ROOT)/gen/firebird/scripts - cp ../builds/install/arch-specific/darwin/install-script \ - $(ROOT)/gen/firebird/scripts/postinstall - cp ../builds/install/arch-specific/darwin/preupgrade-script \ - $(ROOT)/gen/firebird/scripts/preinstall - chmod u+x $(ROOT)/gen/firebird/scripts/postinstall - chmod u+x $(ROOT)/gen/firebird/scripts/preinstall - - rm -fr $(ROOT)/gen/firebird/resources - mkdir $(ROOT)/gen/firebird/resources - cp ../builds/install/arch-specific/darwin/Welcome.txt \ - $(ROOT)/gen/firebird/resources/Welcome.txt - cp ../builds/install/arch-specific/darwin/License.txt \ - $(ROOT)/gen/firebird/resources/License.txt - cp ../builds/install/arch-specific/darwin/DistributionSS.xml \ - $(ROOT)/gen/firebird/DistributionSS.xml - - rm -fr firebird/packages - mkdir firebird/packages - pkgbuild --root $(ROOT)/gen/firebird/frameworks/FirebirdSS.framework \ - --identifier com.firebirdsql.Firebird \ - --install-location /Library/Frameworks/Firebird.framework \ - --scripts $(ROOT)/gen/firebird/scripts \ - firebird/packages/FirebirdSS.pkg - - productbuild --distribution firebird/DistributionSS.xml \ - --resources firebird/resources \ - --package-path firebird/packages \ - firebird/$(INST_NM).pkg - -# rm firebird/firebirdSS.pkg - -install: install_@FIREBIRD_ARCH_TYPE@ - - -install_super: - sudo installer -verbose -dumplog -pkg \ - firebird/FirebirdSS-$(FB_MAJOR_VERS).$(FB_MINOR_VERS).$(FB_REV_NO)-$(FB_BUILD_NO)-$(FB_PLATFORM).pkg -target / - -install_classic: - sudo installer -verbose -dumplog -pkg \ - firebird/FirebirdCS-$(FB_MAJOR_VERS).$(FB_MINOR_VERS).$(FB_REV_NO)-$(FB_BUILD_NO)-$(FB_PLATFORM).pkg -target / diff --git a/builds/install/arch-specific/darwin/Readme.txt b/builds/install/arch-specific/darwin/Readme.txt index f5b8b019ed..bddb22767c 100644 --- a/builds/install/arch-specific/darwin/Readme.txt +++ b/builds/install/arch-specific/darwin/Readme.txt @@ -1,14 +1,15 @@ -The Firebird install process will create a new user: firebird. This is for added security. Please don't delete this user unless you know what you are doing. The installer installs the Firebird framework in /Library/Frameworks. The default installs one super-user database user: "sysdba", password "masterkey". You should change that password using gsec according to the documentation. +The Firebird install process will create a new user: firebird. This is for added security. Please don't delete this user. The installer creates a Firebird framework in /Library/Frameworks. By default the install creates one super-user database user "SYSDBA" with a password "masterkey". You should change that password using gsec or isql according to the documentation. -All the standard command line executables are installed in /Library/Frameworks/Firebird.framework/Resources/bin. If you are interested in helping with the Firebird Project please contact us via the Firebird website at www.firebirdsql.org. +All the standard command line executables are installed in /Library/Frameworks/Firebird.framework/Resources/bin. -Please note that every MacOS X user you want to have access to your database MUST have read/write permissions on the .fdb file. +If you are interested in helping with the Firebird Project please contact us via the Firebird website at www.firebirdsql.org. -The release notes can be found in the doc directory Generic documentation for Firebird can be found on the IBPhoenix web site at www.ibphoenix.com, as well as at the Firebird website. There is also a yahoo group named "ib-support" if you have any problems with Firebird. +The release notes can be found in the doc directory. More generic documentation for Firebird can be found on www.firebirdsql.org and the IBPhoenix web site at www.ibphoenix.com. There is also a yahoo group that can be subscribed to named "ib-support" if you have any problems. Thanks to: John Bellardo (Original MacOSX port for Firebird) David Pugh (Firebird 1.5.3 Port) -Paul Beach & Alex Peshkov (Firebird 1.5.x & 2.x Ports) +Paul Beach & Alex Peshkov (Firebird 1.5.x,2.x & 3.x Ports) +Paul Beach (Firebird Installer) Daniel Puckett (Firebird 2.x Launch Daemon for MacOSX 10.5+) Craig Altenburg (Improvements to the install scripts) diff --git a/builds/install/arch-specific/darwin/Welcome.txt b/builds/install/arch-specific/darwin/Welcome.txt index f1c88fccc4..bb52a7e437 100644 --- a/builds/install/arch-specific/darwin/Welcome.txt +++ b/builds/install/arch-specific/darwin/Welcome.txt @@ -1,5 +1,5 @@ Thank you for choosing the Firebird relational database engine. -Firebird is a relational database offering many ANSI SQL-92 features that runs on Linux, Windows, and a variety of Unix platforms. Firebird offers excellent concurrency, high performance, and powerful language support for stored procedures and triggers. It has been used in production systems, under a variety of names since 1981. +Firebird is a relational database offering many ANSI SQL-2003 features that runs on Linux, Windows, MacOSX and a variety of other Unix platforms. Firebird offers excellent concurrency, high performance, and powerful language support for stored procedures and triggers. It has been used in production systems, under a variety of names since 1981. -Firebird is a commercially independent project of C and C++ programmers, technical advisors and supporters developing and enhancing a multi-platform relational database management system based on the source code released by Inprise Corp (now known as Borland Software Corp) under the InterBase Public License v.1.0 on 25 July, 2000. +Firebird is a commercially independent project of C and C++ programmers, technical advisors and supporters developing and enhancing a multi-platform relational database management system based on the source code released by Borland under the InterBase Public License on the 25th July, 2000. diff --git a/builds/install/arch-specific/darwin/changeServerMode b/builds/install/arch-specific/darwin/changeServerMode new file mode 100755 index 0000000000..321ec92d8c --- /dev/null +++ b/builds/install/arch-specific/darwin/changeServerMode @@ -0,0 +1,134 @@ +#!/bin/sh +# +# The contents of this file are subject to the Initial +# Developer's Public License Version 1.0 (the "License"); +# you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. +# +# Software distributed under the License is distributed AS IS, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. +# See the License for the specific language governing rights +# and limitations under the License. +# +# The Original Code was created by Paul Beach +# based on the original Posix script created by Alex Peshkov +# for the Firebird Open Source RDBMS project. +# +# Copyright (c) 2017 Paul Beach +# Alex Peshkov +# and all contributors signed below. +# +# All Rights Reserved. +# Contributor(s): ______________________________________. +# + +FB_FW=/Library/Frameworks/Firebird.framework +DAEMONLOC=/Library/LaunchDaemons + +#------------------------------------------------------------------------ +# add a line in the (usually) /etc/services or /etc/inetd.conf file +# Here there are three cases, not found => add +# found & different => replace +# found & same => do nothing +# + +replaceLineInFile() { + FileName="$1" + echo $Filename + newLine="$2" + oldLine=`grep "$3" $FileName` + + if [ -z "$oldLine" ] + then + echo "$newLine" >> "$FileName" + elif [ "$oldLine" != "$newLine" ] + then + MakeTemp + grep -v "$oldLine" "$FileName" > "$TmpFile" + echo "$newLine" >> $TmpFile + cp $TmpFile $FileName + rm -f $TmpFile + echo "Updated $1" + fi +} + +MakeTemp() { + TmpFile=`mktemp -q /tmp/firebird_tmp.XXXXXX` + if [ $? -ne 0 ] + then + for n in `seq 1000` + do + TmpFile=/tmp/firebird_tmp.$n + if [ ! -e $TmpFile ] + then + touch $TmpFile + return + fi + done + fi +} + +cat < - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -104,10 +100,6 @@ _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -120,10 +112,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -133,10 +121,6 @@ Disabled _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - diff --git a/builds/win32/msvc10/codes.vcxproj b/builds/win32/msvc10/codes.vcxproj index 6102d558e3..ee53cfde54 100644 --- a/builds/win32/msvc10/codes.vcxproj +++ b/builds/win32/msvc10/codes.vcxproj @@ -100,10 +100,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -122,10 +118,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -144,10 +136,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -168,10 +156,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/common.vcxproj b/builds/win32/msvc10/common.vcxproj index f8f9271413..8c67f99135 100644 --- a/builds/win32/msvc10/common.vcxproj +++ b/builds/win32/msvc10/common.vcxproj @@ -297,10 +297,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) @@ -313,10 +309,6 @@ EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) @@ -332,10 +324,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) @@ -350,10 +338,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) diff --git a/builds/win32/msvc10/empbuild.vcxproj b/builds/win32/msvc10/empbuild.vcxproj index 83a53887c4..49ab17d7d4 100644 --- a/builds/win32/msvc10/empbuild.vcxproj +++ b/builds/win32/msvc10/empbuild.vcxproj @@ -99,10 +99,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;%(AdditionalDependencies) @@ -129,10 +125,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - Console @@ -148,10 +140,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;fbclient.lib;%(AdditionalDependencies) @@ -173,10 +161,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - odbc32.lib;odbccp32.lib;fbclient.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\fbclient;%(AdditionalLibraryDirectories) diff --git a/builds/win32/msvc10/engine.vcxproj b/builds/win32/msvc10/engine.vcxproj index 046601ae35..a35c0a9542 100644 --- a/builds/win32/msvc10/engine.vcxproj +++ b/builds/win32/msvc10/engine.vcxproj @@ -475,10 +475,6 @@ WIN32;_DEBUG;_WINDOWS;_LIB;SUPERSERVER;DEV_BUILD;NAMESPACE=Vulcan;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) @@ -495,10 +491,6 @@ ../../../src/include;../../../src/include/gen;../../../extern/icu/include;../../../src/vulcan;%(AdditionalIncludeDirectories) WIN32;_DEBUG;_WINDOWS;_LIB;SUPERSERVER;DEV_BUILD;NAMESPACE=Vulcan;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) @@ -516,10 +508,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) @@ -541,10 +529,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) diff --git a/builds/win32/msvc10/fb2control.vcxproj b/builds/win32/msvc10/fb2control.vcxproj index 21fa9d29f0..9156de774f 100644 --- a/builds/win32/msvc10/fb2control.vcxproj +++ b/builds/win32/msvc10/fb2control.vcxproj @@ -119,8 +119,7 @@ true - _AFXDLL;NDEBUG;%(PreprocessorDefinitions) - 0x040c + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) @@ -150,8 +149,7 @@ true - _AFXDLL;NDEBUG;%(PreprocessorDefinitions) - 0x040c + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) @@ -180,8 +178,7 @@ EditAndContinue - _AFXDLL;_DEBUG;%(PreprocessorDefinitions) - 0x0809 + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) @@ -207,8 +204,7 @@ TRACE;%(UndefinePreprocessorDefinitions) - _AFXDLL;_DEBUG;%(PreprocessorDefinitions) - 0x0809 + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc10/fb_lock_print.vcxproj b/builds/win32/msvc10/fb_lock_print.vcxproj index bc237dbe3f..a4a981968a 100644 --- a/builds/win32/msvc10/fb_lock_print.vcxproj +++ b/builds/win32/msvc10/fb_lock_print.vcxproj @@ -101,10 +101,6 @@ Speed NDEBUG;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -119,10 +115,6 @@ _DEBUG;DEV_BUILD;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -142,10 +134,6 @@ Speed NDEBUG;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -163,10 +151,6 @@ Disabled _DEBUG;DEV_BUILD;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/fbguard.vcxproj b/builds/win32/msvc10/fbguard.vcxproj index 247ab0a42a..279767ae05 100644 --- a/builds/win32/msvc10/fbguard.vcxproj +++ b/builds/win32/msvc10/fbguard.vcxproj @@ -106,10 +106,6 @@ Speed NDEBUG;WIN32;_WINDOWS;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows @@ -131,10 +127,6 @@ Speed NDEBUG;WIN32;_WINDOWS;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows @@ -155,10 +147,6 @@ _DEBUG;WIN32;_WINDOWS;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows @@ -177,10 +165,6 @@ Disabled _DEBUG;WIN32;_WINDOWS;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows diff --git a/builds/win32/msvc10/fbrmclib.vcxproj b/builds/win32/msvc10/fbrmclib.vcxproj index 27744ffa09..62e85f90c2 100644 --- a/builds/win32/msvc10/fbrmclib.vcxproj +++ b/builds/win32/msvc10/fbrmclib.vcxproj @@ -67,10 +67,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - /MACHINE:I386 /SECTION:.edata,RD %(AdditionalOptions) comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) @@ -78,6 +74,7 @@ false + Windows @@ -94,10 +91,6 @@ NDEBUG;_WINDOWS;_USRDLL;CLIENT;SUPERCLIENT;I386;_X86_=1;WIN32;_X86_;GDS32_EXPORTS;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - /MACHINE:I386 /SECTION:.edata,RD %(AdditionalOptions) comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) @@ -106,6 +99,7 @@ $(OutDir)$(TargetName)$(TargetExt) + Windows diff --git a/builds/win32/msvc10/fbserver.vcxproj b/builds/win32/msvc10/fbserver.vcxproj index 66d0bc86f0..af1c19aeb9 100644 --- a/builds/win32/msvc10/fbserver.vcxproj +++ b/builds/win32/msvc10/fbserver.vcxproj @@ -111,10 +111,6 @@ NDEBUG;_WINDOWS;SUPERSERVER;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) @@ -139,10 +135,6 @@ NDEBUG;_WINDOWS;SUPERSERVER;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) @@ -166,10 +158,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) @@ -191,10 +179,6 @@ _DEBUG;_WINDOWS;SUPERSERVER;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) diff --git a/builds/win32/msvc10/fbsvcmgr.vcxproj b/builds/win32/msvc10/fbsvcmgr.vcxproj index 527cbcf5e5..5e63ac494f 100644 --- a/builds/win32/msvc10/fbsvcmgr.vcxproj +++ b/builds/win32/msvc10/fbsvcmgr.vcxproj @@ -99,10 +99,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -119,10 +115,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -140,10 +132,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -163,10 +151,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/fbtracemgr.vcxproj b/builds/win32/msvc10/fbtracemgr.vcxproj index 66ecd7ee64..3e80050445 100644 --- a/builds/win32/msvc10/fbtracemgr.vcxproj +++ b/builds/win32/msvc10/fbtracemgr.vcxproj @@ -101,10 +101,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -121,10 +117,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -142,10 +134,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -165,10 +153,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/gbak.vcxproj b/builds/win32/msvc10/gbak.vcxproj index c1568bd0c4..ec979a8f42 100644 --- a/builds/win32/msvc10/gbak.vcxproj +++ b/builds/win32/msvc10/gbak.vcxproj @@ -100,10 +100,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -121,10 +117,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -142,10 +134,6 @@ _DEBUG;DEV_BUILD;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -167,10 +155,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/gfix.vcxproj b/builds/win32/msvc10/gfix.vcxproj index fc28c100e4..db95a0b09d 100644 --- a/builds/win32/msvc10/gfix.vcxproj +++ b/builds/win32/msvc10/gfix.vcxproj @@ -100,10 +100,6 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -120,10 +116,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -140,10 +132,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -164,10 +152,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/gpre.vcxproj b/builds/win32/msvc10/gpre.vcxproj index 870ba7ea03..8253380b83 100644 --- a/builds/win32/msvc10/gpre.vcxproj +++ b/builds/win32/msvc10/gpre.vcxproj @@ -103,10 +103,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -128,10 +124,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -148,10 +140,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -170,10 +158,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/gpre_boot.vcxproj b/builds/win32/msvc10/gpre_boot.vcxproj index 3ee0b3b079..731e283c15 100644 --- a/builds/win32/msvc10/gpre_boot.vcxproj +++ b/builds/win32/msvc10/gpre_boot.vcxproj @@ -103,10 +103,6 @@ NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -121,10 +117,6 @@ _DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;_WINDOWS;_USRDLL;CLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -144,10 +136,6 @@ Speed NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -166,10 +154,6 @@ _DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;_WINDOWS;_USRDLL;CLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/gpre_common.vcxproj b/builds/win32/msvc10/gpre_common.vcxproj index 7c28cc0020..fe3403b4fa 100644 --- a/builds/win32/msvc10/gpre_common.vcxproj +++ b/builds/win32/msvc10/gpre_common.vcxproj @@ -99,10 +99,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe @@ -125,10 +121,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe @@ -146,10 +138,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe @@ -169,10 +157,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe diff --git a/builds/win32/msvc10/gsec.vcxproj b/builds/win32/msvc10/gsec.vcxproj index e8acae6014..d433d1a660 100644 --- a/builds/win32/msvc10/gsec.vcxproj +++ b/builds/win32/msvc10/gsec.vcxproj @@ -101,10 +101,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -124,10 +120,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -143,10 +135,6 @@ WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -163,10 +151,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/gsplit.vcxproj b/builds/win32/msvc10/gsplit.vcxproj index 7c894fd9ae..97f159590e 100644 --- a/builds/win32/msvc10/gsplit.vcxproj +++ b/builds/win32/msvc10/gsplit.vcxproj @@ -101,10 +101,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -122,10 +118,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -143,10 +135,6 @@ _DEBUG;DEV_BUILD;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -168,10 +156,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/gstat.vcxproj b/builds/win32/msvc10/gstat.vcxproj index 783b3569c1..b517f3f191 100644 --- a/builds/win32/msvc10/gstat.vcxproj +++ b/builds/win32/msvc10/gstat.vcxproj @@ -99,10 +99,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -119,10 +115,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -140,10 +132,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -163,10 +151,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/ib_util.vcxproj b/builds/win32/msvc10/ib_util.vcxproj index 275270fe56..8e9c561b9a 100644 --- a/builds/win32/msvc10/ib_util.vcxproj +++ b/builds/win32/msvc10/ib_util.vcxproj @@ -110,10 +110,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;IB_UTIL_EXPORTS;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def @@ -136,10 +132,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;IB_UTIL_EXPORTS;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def @@ -161,10 +153,6 @@ WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def @@ -184,10 +172,6 @@ Disabled WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def diff --git a/builds/win32/msvc10/instclient.vcxproj b/builds/win32/msvc10/instclient.vcxproj index d60b236d23..bb6afb6c5e 100644 --- a/builds/win32/msvc10/instclient.vcxproj +++ b/builds/win32/msvc10/instclient.vcxproj @@ -103,10 +103,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -123,10 +119,6 @@ WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -148,10 +140,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -170,10 +158,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/builds/win32/msvc10/instreg.vcxproj b/builds/win32/msvc10/instreg.vcxproj index 1ce9680824..6384b68c32 100644 --- a/builds/win32/msvc10/instreg.vcxproj +++ b/builds/win32/msvc10/instreg.vcxproj @@ -100,10 +100,6 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -121,10 +117,6 @@ Speed WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -143,10 +135,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -168,10 +156,6 @@ Speed WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/builds/win32/msvc10/instsvc.vcxproj b/builds/win32/msvc10/instsvc.vcxproj index c410cc29d1..62adefbf05 100644 --- a/builds/win32/msvc10/instsvc.vcxproj +++ b/builds/win32/msvc10/instsvc.vcxproj @@ -102,10 +102,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -122,10 +118,6 @@ WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -146,10 +138,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -168,10 +156,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/builds/win32/msvc10/intl.vcxproj b/builds/win32/msvc10/intl.vcxproj index 540f364393..eb1ae598d7 100644 --- a/builds/win32/msvc10/intl.vcxproj +++ b/builds/win32/msvc10/intl.vcxproj @@ -115,10 +115,6 @@ NDEBUG;_WINDOWS;_USRDLL;INTL_EXPORTS;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll @@ -143,10 +139,6 @@ NDEBUG;_WINDOWS;_USRDLL;INTL_EXPORTS;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll @@ -170,10 +162,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll @@ -195,10 +183,6 @@ _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll diff --git a/builds/win32/msvc10/intlbuild.vcxproj b/builds/win32/msvc10/intlbuild.vcxproj index 7b8c60eff2..f4ee96ca6d 100644 --- a/builds/win32/msvc10/intlbuild.vcxproj +++ b/builds/win32/msvc10/intlbuild.vcxproj @@ -98,10 +98,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) Console @@ -123,10 +119,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) Console @@ -142,10 +134,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;fbclient_ms.lib;%(AdditionalDependencies) @@ -164,10 +152,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;fbclient_ms.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc10/isql.vcxproj b/builds/win32/msvc10/isql.vcxproj index 9250332e8f..39fa00ce20 100644 --- a/builds/win32/msvc10/isql.vcxproj +++ b/builds/win32/msvc10/isql.vcxproj @@ -101,10 +101,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -122,10 +118,6 @@ NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -143,10 +135,6 @@ _DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -168,10 +156,6 @@ NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/nbackup.vcxproj b/builds/win32/msvc10/nbackup.vcxproj index a972b559ee..81d2bcd4fd 100644 --- a/builds/win32/msvc10/nbackup.vcxproj +++ b/builds/win32/msvc10/nbackup.vcxproj @@ -99,10 +99,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -119,10 +115,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -140,10 +132,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -164,10 +152,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/qli.vcxproj b/builds/win32/msvc10/qli.vcxproj index f8fe5b6b06..63d19dd545 100644 --- a/builds/win32/msvc10/qli.vcxproj +++ b/builds/win32/msvc10/qli.vcxproj @@ -99,10 +99,6 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -119,10 +115,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -140,10 +132,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - /MACHINE:I386 %(AdditionalOptions) comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) @@ -165,10 +153,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc10/remote.vcxproj b/builds/win32/msvc10/remote.vcxproj index 5dc1f223c5..062e136c38 100644 --- a/builds/win32/msvc10/remote.vcxproj +++ b/builds/win32/msvc10/remote.vcxproj @@ -91,10 +91,6 @@ _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) @@ -107,10 +103,6 @@ Disabled _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) @@ -123,10 +115,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) @@ -142,10 +130,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc10/udf_compat.vcxproj b/builds/win32/msvc10/udf_compat.vcxproj index 6d68f72717..3647affa44 100644 --- a/builds/win32/msvc10/udf_compat.vcxproj +++ b/builds/win32/msvc10/udf_compat.vcxproj @@ -107,10 +107,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x340a - false @@ -132,10 +128,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x340a - false @@ -156,10 +148,6 @@ WIN32;_DEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x340a - false @@ -178,10 +166,6 @@ Disabled WIN32;_DEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x340a - false diff --git a/builds/win32/msvc10/udr_engine.vcxproj b/builds/win32/msvc10/udr_engine.vcxproj index d1521f7963..fcfdb62f12 100644 --- a/builds/win32/msvc10/udr_engine.vcxproj +++ b/builds/win32/msvc10/udr_engine.vcxproj @@ -108,10 +108,6 @@ _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll @@ -132,10 +128,6 @@ Disabled _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll @@ -160,10 +152,6 @@ Speed NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll @@ -187,10 +175,6 @@ Speed NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll diff --git a/builds/win32/msvc10/udrcpp_example.vcxproj b/builds/win32/msvc10/udrcpp_example.vcxproj index 9a717c2b1d..b182f33275 100644 --- a/builds/win32/msvc10/udrcpp_example.vcxproj +++ b/builds/win32/msvc10/udrcpp_example.vcxproj @@ -116,10 +116,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll @@ -141,10 +137,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll @@ -170,10 +162,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll @@ -198,10 +186,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll diff --git a/builds/win32/msvc10/yvalve.vcxproj b/builds/win32/msvc10/yvalve.vcxproj index 9d2c229adc..dabf0b6e3f 100644 --- a/builds/win32/msvc10/yvalve.vcxproj +++ b/builds/win32/msvc10/yvalve.vcxproj @@ -170,10 +170,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) @@ -188,10 +184,6 @@ EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) @@ -209,10 +201,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) @@ -229,10 +217,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc12/FirebirdCommon.props b/builds/win32/msvc12/FirebirdCommon.props index 47b5b641b3..a3f24757cb 100644 --- a/builds/win32/msvc12/FirebirdCommon.props +++ b/builds/win32/msvc12/FirebirdCommon.props @@ -37,5 +37,8 @@ $(IntDir)$(TargetName).bsc + + RC_ARH_$(Platform);RC_TARGET_$(TargetName);RC_TARGET_NAME=$(TargetName);RC_TARGET_FILENAME=$(TargetFileName);%(PreprocessorDefinitions) + diff --git a/builds/win32/msvc12/FirebirdDebug.props b/builds/win32/msvc12/FirebirdDebug.props index b1d1b364e8..af1b3dfde6 100644 --- a/builds/win32/msvc12/FirebirdDebug.props +++ b/builds/win32/msvc12/FirebirdDebug.props @@ -8,5 +8,8 @@ MultiThreadedDebugDLL EnableFastChecks + + _DEBUG;%(PreprocessorDefinitions) + \ No newline at end of file diff --git a/builds/win32/msvc12/FirebirdRelease.props b/builds/win32/msvc12/FirebirdRelease.props index a43d1b353d..c2857a78b0 100644 --- a/builds/win32/msvc12/FirebirdRelease.props +++ b/builds/win32/msvc12/FirebirdRelease.props @@ -8,5 +8,8 @@ MultiThreadedDLL /Zo %(AdditionalOptions) + + NDEBUG;%(PreprocessorDefinitions) + \ No newline at end of file diff --git a/builds/win32/msvc12/alice.vcxproj b/builds/win32/msvc12/alice.vcxproj index 155bdc70e4..32312d0c37 100644 --- a/builds/win32/msvc12/alice.vcxproj +++ b/builds/win32/msvc12/alice.vcxproj @@ -97,10 +97,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -113,10 +109,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -124,10 +116,6 @@ _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -137,10 +125,6 @@ Disabled _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - diff --git a/builds/win32/msvc12/build_msg.vcxproj b/builds/win32/msvc12/build_msg.vcxproj index f54aa27cf9..8892fafefe 100644 --- a/builds/win32/msvc12/build_msg.vcxproj +++ b/builds/win32/msvc12/build_msg.vcxproj @@ -107,10 +107,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -131,10 +127,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -151,10 +143,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -173,10 +161,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/burp.vcxproj b/builds/win32/msvc12/burp.vcxproj index 7604499c10..4032ce4323 100644 --- a/builds/win32/msvc12/burp.vcxproj +++ b/builds/win32/msvc12/burp.vcxproj @@ -97,10 +97,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -108,10 +104,6 @@ _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -124,10 +116,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -137,10 +125,6 @@ Disabled _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - diff --git a/builds/win32/msvc12/codes.vcxproj b/builds/win32/msvc12/codes.vcxproj index 56dabb1e0f..d6d40af22a 100644 --- a/builds/win32/msvc12/codes.vcxproj +++ b/builds/win32/msvc12/codes.vcxproj @@ -104,10 +104,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -126,10 +122,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -148,10 +140,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -172,10 +160,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/common.vcxproj b/builds/win32/msvc12/common.vcxproj index 8b2e9805a0..449ad3c6ba 100644 --- a/builds/win32/msvc12/common.vcxproj +++ b/builds/win32/msvc12/common.vcxproj @@ -297,10 +297,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) @@ -313,10 +309,6 @@ EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;%(AdditionalDependencies) @@ -332,10 +324,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) @@ -350,10 +338,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) diff --git a/builds/win32/msvc12/empbuild.vcxproj b/builds/win32/msvc12/empbuild.vcxproj index db6850eb5a..e52dc64bd5 100644 --- a/builds/win32/msvc12/empbuild.vcxproj +++ b/builds/win32/msvc12/empbuild.vcxproj @@ -103,10 +103,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;%(AdditionalDependencies) @@ -133,10 +129,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - Console @@ -152,10 +144,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;fbclient.lib;%(AdditionalDependencies) @@ -177,10 +165,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - odbc32.lib;odbccp32.lib;fbclient.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\fbclient;%(AdditionalLibraryDirectories) diff --git a/builds/win32/msvc12/engine.vcxproj b/builds/win32/msvc12/engine.vcxproj index 683779c3e8..6ecc29ecc2 100644 --- a/builds/win32/msvc12/engine.vcxproj +++ b/builds/win32/msvc12/engine.vcxproj @@ -492,10 +492,6 @@ WIN32;_DEBUG;_WINDOWS;_LIB;SUPERSERVER;DEV_BUILD;NAMESPACE=Vulcan;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) @@ -512,10 +508,6 @@ ../../../src/include;../../../src/include/gen;../../../extern/icu/include;../../../src/vulcan;%(AdditionalIncludeDirectories) WIN32;_DEBUG;_WINDOWS;_LIB;SUPERSERVER;DEV_BUILD;NAMESPACE=Vulcan;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) @@ -533,10 +525,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) @@ -558,10 +546,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) diff --git a/builds/win32/msvc12/fb2control.vcxproj b/builds/win32/msvc12/fb2control.vcxproj index 73a43dc4ca..02cc44f90c 100644 --- a/builds/win32/msvc12/fb2control.vcxproj +++ b/builds/win32/msvc12/fb2control.vcxproj @@ -123,8 +123,7 @@ true - _AFXDLL;NDEBUG;%(PreprocessorDefinitions) - 0x040c + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) @@ -154,8 +153,7 @@ true - _AFXDLL;NDEBUG;%(PreprocessorDefinitions) - 0x040c + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) @@ -184,8 +182,7 @@ EditAndContinue - _AFXDLL;_DEBUG;%(PreprocessorDefinitions) - 0x0809 + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) @@ -211,8 +208,7 @@ TRACE;%(UndefinePreprocessorDefinitions) - _AFXDLL;_DEBUG;%(PreprocessorDefinitions) - 0x0809 + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc12/fb_lock_print.vcxproj b/builds/win32/msvc12/fb_lock_print.vcxproj index 10773ca629..948fb12e37 100644 --- a/builds/win32/msvc12/fb_lock_print.vcxproj +++ b/builds/win32/msvc12/fb_lock_print.vcxproj @@ -105,10 +105,6 @@ Speed NDEBUG;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -123,10 +119,6 @@ _DEBUG;DEV_BUILD;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -146,10 +138,6 @@ Speed NDEBUG;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -167,10 +155,6 @@ Disabled _DEBUG;DEV_BUILD;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/fbguard.vcxproj b/builds/win32/msvc12/fbguard.vcxproj index 856779a2f3..c0c1546e90 100644 --- a/builds/win32/msvc12/fbguard.vcxproj +++ b/builds/win32/msvc12/fbguard.vcxproj @@ -110,10 +110,6 @@ Speed NDEBUG;WIN32;_WINDOWS;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows @@ -135,10 +131,6 @@ Speed NDEBUG;WIN32;_WINDOWS;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows @@ -159,10 +151,6 @@ _DEBUG;WIN32;_WINDOWS;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows @@ -181,10 +169,6 @@ Disabled _DEBUG;WIN32;_WINDOWS;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows diff --git a/builds/win32/msvc12/fbrmclib.vcxproj b/builds/win32/msvc12/fbrmclib.vcxproj index b4e93da3c1..25d29bac4f 100644 --- a/builds/win32/msvc12/fbrmclib.vcxproj +++ b/builds/win32/msvc12/fbrmclib.vcxproj @@ -69,10 +69,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - /MACHINE:I386 /SECTION:.edata,RD %(AdditionalOptions) comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) @@ -97,10 +93,6 @@ NDEBUG;_WINDOWS;_USRDLL;CLIENT;SUPERCLIENT;I386;_X86_=1;WIN32;_X86_;GDS32_EXPORTS;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - /MACHINE:I386 /SECTION:.edata,RD %(AdditionalOptions) comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc12/fbserver.vcxproj b/builds/win32/msvc12/fbserver.vcxproj index db5b0e57ac..880b0a1056 100644 --- a/builds/win32/msvc12/fbserver.vcxproj +++ b/builds/win32/msvc12/fbserver.vcxproj @@ -115,10 +115,6 @@ NDEBUG;_WINDOWS;SUPERSERVER;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) @@ -143,10 +139,6 @@ NDEBUG;_WINDOWS;SUPERSERVER;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) @@ -170,10 +162,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) @@ -195,10 +183,6 @@ _DEBUG;_WINDOWS;SUPERSERVER;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) diff --git a/builds/win32/msvc12/fbsvcmgr.vcxproj b/builds/win32/msvc12/fbsvcmgr.vcxproj index 8c2a632cb2..a56a65482b 100644 --- a/builds/win32/msvc12/fbsvcmgr.vcxproj +++ b/builds/win32/msvc12/fbsvcmgr.vcxproj @@ -103,10 +103,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -123,10 +119,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -144,10 +136,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -167,10 +155,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/fbtracemgr.vcxproj b/builds/win32/msvc12/fbtracemgr.vcxproj index 4c4d57a952..f67c7ffbd5 100644 --- a/builds/win32/msvc12/fbtracemgr.vcxproj +++ b/builds/win32/msvc12/fbtracemgr.vcxproj @@ -105,10 +105,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -125,10 +121,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -146,10 +138,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -169,10 +157,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/gbak.vcxproj b/builds/win32/msvc12/gbak.vcxproj index 500afb688a..985618a567 100644 --- a/builds/win32/msvc12/gbak.vcxproj +++ b/builds/win32/msvc12/gbak.vcxproj @@ -104,10 +104,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -125,10 +121,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -146,10 +138,6 @@ _DEBUG;DEV_BUILD;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -171,10 +159,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/gfix.vcxproj b/builds/win32/msvc12/gfix.vcxproj index b474c7ad94..db12bdcbe8 100644 --- a/builds/win32/msvc12/gfix.vcxproj +++ b/builds/win32/msvc12/gfix.vcxproj @@ -104,10 +104,6 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -124,10 +120,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -144,10 +136,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -168,10 +156,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/gpre.vcxproj b/builds/win32/msvc12/gpre.vcxproj index 12a51b4c1c..f5182843d0 100644 --- a/builds/win32/msvc12/gpre.vcxproj +++ b/builds/win32/msvc12/gpre.vcxproj @@ -107,10 +107,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -132,10 +128,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -152,10 +144,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -174,10 +162,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/gpre_boot.vcxproj b/builds/win32/msvc12/gpre_boot.vcxproj index cb305f30d4..51d9d90107 100644 --- a/builds/win32/msvc12/gpre_boot.vcxproj +++ b/builds/win32/msvc12/gpre_boot.vcxproj @@ -107,10 +107,6 @@ NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -125,10 +121,6 @@ _DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;_WINDOWS;_USRDLL;CLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -148,10 +140,6 @@ Speed NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -170,10 +158,6 @@ _DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;_WINDOWS;_USRDLL;CLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/gpre_common.vcxproj b/builds/win32/msvc12/gpre_common.vcxproj index af79ca9ea4..7472559480 100644 --- a/builds/win32/msvc12/gpre_common.vcxproj +++ b/builds/win32/msvc12/gpre_common.vcxproj @@ -103,10 +103,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe @@ -129,10 +125,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe @@ -150,10 +142,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe @@ -173,10 +161,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe diff --git a/builds/win32/msvc12/gsec.vcxproj b/builds/win32/msvc12/gsec.vcxproj index 650aec7631..39c91e24f8 100644 --- a/builds/win32/msvc12/gsec.vcxproj +++ b/builds/win32/msvc12/gsec.vcxproj @@ -105,10 +105,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -128,10 +124,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -147,10 +139,6 @@ WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -167,10 +155,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/gsplit.vcxproj b/builds/win32/msvc12/gsplit.vcxproj index c7ef719231..6656b92843 100644 --- a/builds/win32/msvc12/gsplit.vcxproj +++ b/builds/win32/msvc12/gsplit.vcxproj @@ -105,10 +105,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -126,10 +122,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -147,10 +139,6 @@ _DEBUG;DEV_BUILD;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -172,10 +160,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/gstat.vcxproj b/builds/win32/msvc12/gstat.vcxproj index dfddab8908..ec35a4f54a 100644 --- a/builds/win32/msvc12/gstat.vcxproj +++ b/builds/win32/msvc12/gstat.vcxproj @@ -103,10 +103,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -123,10 +119,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -144,10 +136,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -167,10 +155,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/ib_util.vcxproj b/builds/win32/msvc12/ib_util.vcxproj index 742900db31..69af9b768b 100644 --- a/builds/win32/msvc12/ib_util.vcxproj +++ b/builds/win32/msvc12/ib_util.vcxproj @@ -114,10 +114,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;IB_UTIL_EXPORTS;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def @@ -140,10 +136,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;IB_UTIL_EXPORTS;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def @@ -165,10 +157,6 @@ WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def @@ -188,10 +176,6 @@ Disabled WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def diff --git a/builds/win32/msvc12/instclient.vcxproj b/builds/win32/msvc12/instclient.vcxproj index 25d34b274a..d8a650065f 100644 --- a/builds/win32/msvc12/instclient.vcxproj +++ b/builds/win32/msvc12/instclient.vcxproj @@ -107,10 +107,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -127,10 +123,6 @@ WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -152,10 +144,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -174,10 +162,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/builds/win32/msvc12/instreg.vcxproj b/builds/win32/msvc12/instreg.vcxproj index 7ac1431d5c..b8ebafd24c 100644 --- a/builds/win32/msvc12/instreg.vcxproj +++ b/builds/win32/msvc12/instreg.vcxproj @@ -104,10 +104,6 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -125,10 +121,6 @@ Speed WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -147,10 +139,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -172,10 +160,6 @@ Speed WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/builds/win32/msvc12/instsvc.vcxproj b/builds/win32/msvc12/instsvc.vcxproj index 534c022d6d..3e8e6ed7aa 100644 --- a/builds/win32/msvc12/instsvc.vcxproj +++ b/builds/win32/msvc12/instsvc.vcxproj @@ -106,10 +106,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -126,10 +122,6 @@ WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -150,10 +142,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -172,10 +160,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/builds/win32/msvc12/intl.vcxproj b/builds/win32/msvc12/intl.vcxproj index 615c50e044..3490ef5d03 100644 --- a/builds/win32/msvc12/intl.vcxproj +++ b/builds/win32/msvc12/intl.vcxproj @@ -119,10 +119,6 @@ NDEBUG;_WINDOWS;_USRDLL;INTL_EXPORTS;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll @@ -147,10 +143,6 @@ NDEBUG;_WINDOWS;_USRDLL;INTL_EXPORTS;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll @@ -174,10 +166,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll @@ -199,10 +187,6 @@ _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll diff --git a/builds/win32/msvc12/intlbuild.vcxproj b/builds/win32/msvc12/intlbuild.vcxproj index 62e42de7ba..c62876ff15 100644 --- a/builds/win32/msvc12/intlbuild.vcxproj +++ b/builds/win32/msvc12/intlbuild.vcxproj @@ -102,10 +102,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) Console @@ -127,10 +123,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) Console @@ -146,10 +138,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;fbclient_ms.lib;%(AdditionalDependencies) @@ -168,10 +156,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;fbclient_ms.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc12/isql.vcxproj b/builds/win32/msvc12/isql.vcxproj index 4e31430dbc..be0904957d 100644 --- a/builds/win32/msvc12/isql.vcxproj +++ b/builds/win32/msvc12/isql.vcxproj @@ -105,10 +105,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -126,10 +122,6 @@ NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -147,10 +139,6 @@ _DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -172,10 +160,6 @@ NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/nbackup.vcxproj b/builds/win32/msvc12/nbackup.vcxproj index 13ea3132df..b3ede52a65 100644 --- a/builds/win32/msvc12/nbackup.vcxproj +++ b/builds/win32/msvc12/nbackup.vcxproj @@ -103,10 +103,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -123,10 +119,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -144,10 +136,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -168,10 +156,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/qli.vcxproj b/builds/win32/msvc12/qli.vcxproj index 19bb81bf9b..1b08787531 100644 --- a/builds/win32/msvc12/qli.vcxproj +++ b/builds/win32/msvc12/qli.vcxproj @@ -103,10 +103,6 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -123,10 +119,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -144,10 +136,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - /MACHINE:I386 %(AdditionalOptions) comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) @@ -169,10 +157,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc12/remote.vcxproj b/builds/win32/msvc12/remote.vcxproj index 850225ce52..481a1b3b89 100644 --- a/builds/win32/msvc12/remote.vcxproj +++ b/builds/win32/msvc12/remote.vcxproj @@ -95,10 +95,6 @@ _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) @@ -111,10 +107,6 @@ Disabled _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) @@ -127,10 +119,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) @@ -146,10 +134,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc12/udf_compat.vcxproj b/builds/win32/msvc12/udf_compat.vcxproj index 457a4f77bf..15cf0285db 100644 --- a/builds/win32/msvc12/udf_compat.vcxproj +++ b/builds/win32/msvc12/udf_compat.vcxproj @@ -111,10 +111,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x340a - false @@ -136,10 +132,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x340a - false @@ -160,10 +152,6 @@ WIN32;_DEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x340a - false @@ -182,10 +170,6 @@ Disabled WIN32;_DEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x340a - false diff --git a/builds/win32/msvc12/udr_engine.vcxproj b/builds/win32/msvc12/udr_engine.vcxproj index 2a6d14e613..38a7404c3d 100644 --- a/builds/win32/msvc12/udr_engine.vcxproj +++ b/builds/win32/msvc12/udr_engine.vcxproj @@ -112,10 +112,6 @@ _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll @@ -136,10 +132,6 @@ Disabled _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll @@ -164,10 +156,6 @@ Speed NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll @@ -191,10 +179,6 @@ Speed NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll diff --git a/builds/win32/msvc12/udrcpp_example.vcxproj b/builds/win32/msvc12/udrcpp_example.vcxproj index 5491123aa1..3d3ea95312 100644 --- a/builds/win32/msvc12/udrcpp_example.vcxproj +++ b/builds/win32/msvc12/udrcpp_example.vcxproj @@ -120,10 +120,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll @@ -145,10 +141,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll @@ -174,10 +166,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll @@ -202,10 +190,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll diff --git a/builds/win32/msvc12/yvalve.vcxproj b/builds/win32/msvc12/yvalve.vcxproj index 47b56a8287..77975a407a 100644 --- a/builds/win32/msvc12/yvalve.vcxproj +++ b/builds/win32/msvc12/yvalve.vcxproj @@ -174,10 +174,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) @@ -192,10 +188,6 @@ EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) @@ -213,10 +205,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) @@ -233,10 +221,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc14/FirebirdCommon.props b/builds/win32/msvc14/FirebirdCommon.props index c296233d68..50faef634d 100644 --- a/builds/win32/msvc14/FirebirdCommon.props +++ b/builds/win32/msvc14/FirebirdCommon.props @@ -37,5 +37,8 @@ $(IntDir)$(TargetName).bsc + + RC_ARH_$(Platform);RC_TARGET_$(TargetName);RC_TARGET_NAME=$(TargetName);RC_TARGET_FILENAME=$(TargetFileName);%(PreprocessorDefinitions) + \ No newline at end of file diff --git a/builds/win32/msvc14/FirebirdDebug.props b/builds/win32/msvc14/FirebirdDebug.props index b1d1b364e8..af1b3dfde6 100644 --- a/builds/win32/msvc14/FirebirdDebug.props +++ b/builds/win32/msvc14/FirebirdDebug.props @@ -8,5 +8,8 @@ MultiThreadedDebugDLL EnableFastChecks + + _DEBUG;%(PreprocessorDefinitions) + \ No newline at end of file diff --git a/builds/win32/msvc14/FirebirdRelease.props b/builds/win32/msvc14/FirebirdRelease.props index a43d1b353d..c2857a78b0 100644 --- a/builds/win32/msvc14/FirebirdRelease.props +++ b/builds/win32/msvc14/FirebirdRelease.props @@ -8,5 +8,8 @@ MultiThreadedDLL /Zo %(AdditionalOptions) + + NDEBUG;%(PreprocessorDefinitions) + \ No newline at end of file diff --git a/builds/win32/msvc14/alice.vcxproj b/builds/win32/msvc14/alice.vcxproj index 3befbf9019..7047a027c8 100644 --- a/builds/win32/msvc14/alice.vcxproj +++ b/builds/win32/msvc14/alice.vcxproj @@ -97,10 +97,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -113,10 +109,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -124,10 +116,6 @@ _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -137,10 +125,6 @@ Disabled _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - diff --git a/builds/win32/msvc14/build_msg.vcxproj b/builds/win32/msvc14/build_msg.vcxproj index ff03fb4878..9168f91713 100644 --- a/builds/win32/msvc14/build_msg.vcxproj +++ b/builds/win32/msvc14/build_msg.vcxproj @@ -107,10 +107,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -131,10 +127,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -151,10 +143,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -173,10 +161,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/burp.vcxproj b/builds/win32/msvc14/burp.vcxproj index 1d9c13a156..dec13a3cdc 100644 --- a/builds/win32/msvc14/burp.vcxproj +++ b/builds/win32/msvc14/burp.vcxproj @@ -97,10 +97,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -108,10 +104,6 @@ _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -124,10 +116,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -137,10 +125,6 @@ Disabled _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - diff --git a/builds/win32/msvc14/codes.vcxproj b/builds/win32/msvc14/codes.vcxproj index aa90e17b62..46a9048c07 100644 --- a/builds/win32/msvc14/codes.vcxproj +++ b/builds/win32/msvc14/codes.vcxproj @@ -104,10 +104,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -126,10 +122,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -148,10 +140,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -172,10 +160,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/common.vcxproj b/builds/win32/msvc14/common.vcxproj index 615a314773..ff4e726c80 100644 --- a/builds/win32/msvc14/common.vcxproj +++ b/builds/win32/msvc14/common.vcxproj @@ -299,10 +299,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) @@ -315,10 +311,6 @@ EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) @@ -334,10 +326,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) @@ -352,10 +340,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) diff --git a/builds/win32/msvc14/empbuild.vcxproj b/builds/win32/msvc14/empbuild.vcxproj index 9f9e87a5c1..b6bc0e0a4c 100644 --- a/builds/win32/msvc14/empbuild.vcxproj +++ b/builds/win32/msvc14/empbuild.vcxproj @@ -103,10 +103,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;%(AdditionalDependencies) @@ -133,10 +129,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - Console @@ -152,10 +144,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;fbclient.lib;%(AdditionalDependencies) @@ -177,10 +165,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - odbc32.lib;odbccp32.lib;fbclient.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\fbclient;%(AdditionalLibraryDirectories) diff --git a/builds/win32/msvc14/engine.vcxproj b/builds/win32/msvc14/engine.vcxproj index 43e916350e..0ef26d8ee7 100644 --- a/builds/win32/msvc14/engine.vcxproj +++ b/builds/win32/msvc14/engine.vcxproj @@ -491,10 +491,6 @@ WIN32;_DEBUG;_WINDOWS;_LIB;SUPERSERVER;DEV_BUILD;NAMESPACE=Vulcan;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) @@ -511,10 +507,6 @@ ../../../src/include;../../../src/include/gen;../../../extern/icu/include;../../../src/vulcan;%(AdditionalIncludeDirectories) WIN32;_DEBUG;_WINDOWS;_LIB;SUPERSERVER;DEV_BUILD;NAMESPACE=Vulcan;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) @@ -532,10 +524,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) @@ -557,10 +545,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) diff --git a/builds/win32/msvc14/fb2control.vcxproj b/builds/win32/msvc14/fb2control.vcxproj index d0d1508c18..98bcb3ae39 100644 --- a/builds/win32/msvc14/fb2control.vcxproj +++ b/builds/win32/msvc14/fb2control.vcxproj @@ -123,8 +123,7 @@ true - _AFXDLL;NDEBUG;%(PreprocessorDefinitions) - 0x040c + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) @@ -154,8 +153,7 @@ true - _AFXDLL;NDEBUG;%(PreprocessorDefinitions) - 0x040c + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) @@ -184,8 +182,7 @@ EditAndContinue - _AFXDLL;_DEBUG;%(PreprocessorDefinitions) - 0x0809 + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) @@ -211,8 +208,7 @@ TRACE;%(UndefinePreprocessorDefinitions) - _AFXDLL;_DEBUG;%(PreprocessorDefinitions) - 0x0809 + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc14/fb_lock_print.vcxproj b/builds/win32/msvc14/fb_lock_print.vcxproj index 6ad1f4874b..44ace81c7b 100644 --- a/builds/win32/msvc14/fb_lock_print.vcxproj +++ b/builds/win32/msvc14/fb_lock_print.vcxproj @@ -105,10 +105,6 @@ Speed NDEBUG;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -123,10 +119,6 @@ _DEBUG;DEV_BUILD;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -146,10 +138,6 @@ Speed NDEBUG;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -167,10 +155,6 @@ Disabled _DEBUG;DEV_BUILD;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/fbguard.vcxproj b/builds/win32/msvc14/fbguard.vcxproj index 393440800c..944821aaf0 100644 --- a/builds/win32/msvc14/fbguard.vcxproj +++ b/builds/win32/msvc14/fbguard.vcxproj @@ -110,10 +110,6 @@ Speed NDEBUG;WIN32;_WINDOWS;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows @@ -135,10 +131,6 @@ Speed NDEBUG;WIN32;_WINDOWS;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows @@ -159,10 +151,6 @@ _DEBUG;WIN32;_WINDOWS;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows @@ -181,10 +169,6 @@ Disabled _DEBUG;WIN32;_WINDOWS;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows diff --git a/builds/win32/msvc14/fbrmclib.vcxproj b/builds/win32/msvc14/fbrmclib.vcxproj index 10dffafd9c..e63efe126c 100644 --- a/builds/win32/msvc14/fbrmclib.vcxproj +++ b/builds/win32/msvc14/fbrmclib.vcxproj @@ -69,10 +69,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - /MACHINE:I386 /SECTION:.edata,RD %(AdditionalOptions) comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) @@ -97,10 +93,6 @@ NDEBUG;_WINDOWS;_USRDLL;CLIENT;SUPERCLIENT;I386;_X86_=1;WIN32;_X86_;GDS32_EXPORTS;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - /MACHINE:I386 /SECTION:.edata,RD %(AdditionalOptions) comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc14/fbserver.vcxproj b/builds/win32/msvc14/fbserver.vcxproj index 0a9b234dd7..463065149c 100644 --- a/builds/win32/msvc14/fbserver.vcxproj +++ b/builds/win32/msvc14/fbserver.vcxproj @@ -115,10 +115,6 @@ NDEBUG;_WINDOWS;SUPERSERVER;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) @@ -143,10 +139,6 @@ NDEBUG;_WINDOWS;SUPERSERVER;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) @@ -170,10 +162,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) @@ -195,10 +183,6 @@ _DEBUG;_WINDOWS;SUPERSERVER;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) diff --git a/builds/win32/msvc14/fbsvcmgr.vcxproj b/builds/win32/msvc14/fbsvcmgr.vcxproj index dd86a3ca8f..aaae030287 100644 --- a/builds/win32/msvc14/fbsvcmgr.vcxproj +++ b/builds/win32/msvc14/fbsvcmgr.vcxproj @@ -103,10 +103,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -123,10 +119,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -144,10 +136,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -167,10 +155,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/fbtracemgr.vcxproj b/builds/win32/msvc14/fbtracemgr.vcxproj index 86d486c506..c5a05fb4e6 100644 --- a/builds/win32/msvc14/fbtracemgr.vcxproj +++ b/builds/win32/msvc14/fbtracemgr.vcxproj @@ -105,10 +105,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -125,10 +121,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -146,10 +138,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -169,10 +157,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/gbak.vcxproj b/builds/win32/msvc14/gbak.vcxproj index 9664fd410c..b26e0b7441 100644 --- a/builds/win32/msvc14/gbak.vcxproj +++ b/builds/win32/msvc14/gbak.vcxproj @@ -104,10 +104,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -125,10 +121,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -146,10 +138,6 @@ _DEBUG;DEV_BUILD;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -171,10 +159,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/gfix.vcxproj b/builds/win32/msvc14/gfix.vcxproj index ca4383c379..72c47dc23d 100644 --- a/builds/win32/msvc14/gfix.vcxproj +++ b/builds/win32/msvc14/gfix.vcxproj @@ -104,10 +104,6 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -124,10 +120,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -144,10 +136,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -168,10 +156,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/gpre.vcxproj b/builds/win32/msvc14/gpre.vcxproj index 9a0270c913..6ce7f7b9f7 100644 --- a/builds/win32/msvc14/gpre.vcxproj +++ b/builds/win32/msvc14/gpre.vcxproj @@ -107,10 +107,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -132,10 +128,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -152,10 +144,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -174,10 +162,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/gpre_boot.vcxproj b/builds/win32/msvc14/gpre_boot.vcxproj index da65a90b04..17044b7e89 100644 --- a/builds/win32/msvc14/gpre_boot.vcxproj +++ b/builds/win32/msvc14/gpre_boot.vcxproj @@ -107,10 +107,6 @@ NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -125,10 +121,6 @@ _DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;_WINDOWS;_USRDLL;CLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -148,10 +140,6 @@ Speed NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -170,10 +158,6 @@ _DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;_WINDOWS;_USRDLL;CLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/gpre_common.vcxproj b/builds/win32/msvc14/gpre_common.vcxproj index 14025e0958..74966fc8f8 100644 --- a/builds/win32/msvc14/gpre_common.vcxproj +++ b/builds/win32/msvc14/gpre_common.vcxproj @@ -103,10 +103,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe @@ -129,10 +125,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe @@ -150,10 +142,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe @@ -173,10 +161,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe diff --git a/builds/win32/msvc14/gsec.vcxproj b/builds/win32/msvc14/gsec.vcxproj index 35a02f1adb..3087b495ef 100644 --- a/builds/win32/msvc14/gsec.vcxproj +++ b/builds/win32/msvc14/gsec.vcxproj @@ -105,10 +105,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -128,10 +124,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -147,10 +139,6 @@ WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -167,10 +155,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/gsplit.vcxproj b/builds/win32/msvc14/gsplit.vcxproj index 8538614d1d..d82e18726c 100644 --- a/builds/win32/msvc14/gsplit.vcxproj +++ b/builds/win32/msvc14/gsplit.vcxproj @@ -105,10 +105,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -126,10 +122,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -147,10 +139,6 @@ _DEBUG;DEV_BUILD;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -172,10 +160,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/gstat.vcxproj b/builds/win32/msvc14/gstat.vcxproj index db0f0ce2c7..0520244e54 100644 --- a/builds/win32/msvc14/gstat.vcxproj +++ b/builds/win32/msvc14/gstat.vcxproj @@ -103,10 +103,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -123,10 +119,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -144,10 +136,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -167,10 +155,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/ib_util.vcxproj b/builds/win32/msvc14/ib_util.vcxproj index 5c71abd45b..810aebf82e 100644 --- a/builds/win32/msvc14/ib_util.vcxproj +++ b/builds/win32/msvc14/ib_util.vcxproj @@ -114,10 +114,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;IB_UTIL_EXPORTS;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def @@ -140,10 +136,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;IB_UTIL_EXPORTS;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def @@ -165,10 +157,6 @@ WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def @@ -188,10 +176,6 @@ Disabled WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def diff --git a/builds/win32/msvc14/instclient.vcxproj b/builds/win32/msvc14/instclient.vcxproj index 27a70ea4b7..01b56c0751 100644 --- a/builds/win32/msvc14/instclient.vcxproj +++ b/builds/win32/msvc14/instclient.vcxproj @@ -107,10 +107,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -127,10 +123,6 @@ WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -152,10 +144,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -174,10 +162,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/builds/win32/msvc14/instreg.vcxproj b/builds/win32/msvc14/instreg.vcxproj index f1ad71557c..3ef918e985 100644 --- a/builds/win32/msvc14/instreg.vcxproj +++ b/builds/win32/msvc14/instreg.vcxproj @@ -104,10 +104,6 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -125,10 +121,6 @@ Speed WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -147,10 +139,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -172,10 +160,6 @@ Speed WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/builds/win32/msvc14/instsvc.vcxproj b/builds/win32/msvc14/instsvc.vcxproj index 706c36fef0..0b907a8555 100644 --- a/builds/win32/msvc14/instsvc.vcxproj +++ b/builds/win32/msvc14/instsvc.vcxproj @@ -106,10 +106,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -126,10 +122,6 @@ WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -150,10 +142,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -172,10 +160,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/builds/win32/msvc14/intl.vcxproj b/builds/win32/msvc14/intl.vcxproj index 7788a7acbf..e0ae4d762e 100644 --- a/builds/win32/msvc14/intl.vcxproj +++ b/builds/win32/msvc14/intl.vcxproj @@ -119,10 +119,6 @@ NDEBUG;_WINDOWS;_USRDLL;INTL_EXPORTS;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll @@ -147,10 +143,6 @@ NDEBUG;_WINDOWS;_USRDLL;INTL_EXPORTS;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll @@ -174,10 +166,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll @@ -199,10 +187,6 @@ _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll diff --git a/builds/win32/msvc14/intlbuild.vcxproj b/builds/win32/msvc14/intlbuild.vcxproj index cd87f7035e..f3ff7c643e 100644 --- a/builds/win32/msvc14/intlbuild.vcxproj +++ b/builds/win32/msvc14/intlbuild.vcxproj @@ -102,10 +102,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) Console @@ -127,10 +123,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) Console @@ -146,10 +138,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;fbclient_ms.lib;%(AdditionalDependencies) @@ -168,10 +156,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;fbclient_ms.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc14/isql.vcxproj b/builds/win32/msvc14/isql.vcxproj index 4cf62655c6..24b390c047 100644 --- a/builds/win32/msvc14/isql.vcxproj +++ b/builds/win32/msvc14/isql.vcxproj @@ -105,10 +105,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -126,10 +122,6 @@ NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -147,10 +139,6 @@ _DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -172,10 +160,6 @@ NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/nbackup.vcxproj b/builds/win32/msvc14/nbackup.vcxproj index 87f2104540..abb1f18d23 100644 --- a/builds/win32/msvc14/nbackup.vcxproj +++ b/builds/win32/msvc14/nbackup.vcxproj @@ -103,10 +103,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -123,10 +119,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -144,10 +136,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -168,10 +156,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/qli.vcxproj b/builds/win32/msvc14/qli.vcxproj index 4fafea41f9..193c547949 100644 --- a/builds/win32/msvc14/qli.vcxproj +++ b/builds/win32/msvc14/qli.vcxproj @@ -103,10 +103,6 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -123,10 +119,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -144,10 +136,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - /MACHINE:I386 %(AdditionalOptions) comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) @@ -169,10 +157,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc14/remote.vcxproj b/builds/win32/msvc14/remote.vcxproj index 9162277473..aed230ec2f 100644 --- a/builds/win32/msvc14/remote.vcxproj +++ b/builds/win32/msvc14/remote.vcxproj @@ -95,10 +95,6 @@ _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) @@ -111,10 +107,6 @@ Disabled _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) @@ -127,10 +119,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) @@ -146,10 +134,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc14/udf_compat.vcxproj b/builds/win32/msvc14/udf_compat.vcxproj index dd1915681e..9612d56282 100644 --- a/builds/win32/msvc14/udf_compat.vcxproj +++ b/builds/win32/msvc14/udf_compat.vcxproj @@ -111,10 +111,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x340a - false @@ -136,10 +132,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x340a - false @@ -160,10 +152,6 @@ WIN32;_DEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x340a - false @@ -182,10 +170,6 @@ Disabled WIN32;_DEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x340a - false diff --git a/builds/win32/msvc14/udr_engine.vcxproj b/builds/win32/msvc14/udr_engine.vcxproj index bd31b6be06..21fe28af24 100644 --- a/builds/win32/msvc14/udr_engine.vcxproj +++ b/builds/win32/msvc14/udr_engine.vcxproj @@ -112,10 +112,6 @@ _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll @@ -136,10 +132,6 @@ Disabled _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll @@ -164,10 +156,6 @@ Speed NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll @@ -191,10 +179,6 @@ Speed NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll diff --git a/builds/win32/msvc14/udrcpp_example.vcxproj b/builds/win32/msvc14/udrcpp_example.vcxproj index 205ef24e52..01ead89223 100644 --- a/builds/win32/msvc14/udrcpp_example.vcxproj +++ b/builds/win32/msvc14/udrcpp_example.vcxproj @@ -120,10 +120,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll @@ -145,10 +141,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll @@ -174,10 +166,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll @@ -202,10 +190,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll diff --git a/builds/win32/msvc14/yvalve.vcxproj b/builds/win32/msvc14/yvalve.vcxproj index 51d1d4c600..181e4eab7a 100644 --- a/builds/win32/msvc14/yvalve.vcxproj +++ b/builds/win32/msvc14/yvalve.vcxproj @@ -174,10 +174,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) @@ -192,10 +188,6 @@ EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) @@ -213,10 +205,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) @@ -233,10 +221,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc15/FirebirdCommon.props b/builds/win32/msvc15/FirebirdCommon.props index 5c24a0063f..50faef634d 100644 --- a/builds/win32/msvc15/FirebirdCommon.props +++ b/builds/win32/msvc15/FirebirdCommon.props @@ -37,5 +37,8 @@ $(IntDir)$(TargetName).bsc + + RC_ARH_$(Platform);RC_TARGET_$(TargetName);RC_TARGET_NAME=$(TargetName);RC_TARGET_FILENAME=$(TargetFileName);%(PreprocessorDefinitions) + - + \ No newline at end of file diff --git a/builds/win32/msvc15/FirebirdDebug.props b/builds/win32/msvc15/FirebirdDebug.props index b1d1b364e8..af1b3dfde6 100644 --- a/builds/win32/msvc15/FirebirdDebug.props +++ b/builds/win32/msvc15/FirebirdDebug.props @@ -8,5 +8,8 @@ MultiThreadedDebugDLL EnableFastChecks + + _DEBUG;%(PreprocessorDefinitions) + \ No newline at end of file diff --git a/builds/win32/msvc15/FirebirdRelease.props b/builds/win32/msvc15/FirebirdRelease.props index a43d1b353d..c2857a78b0 100644 --- a/builds/win32/msvc15/FirebirdRelease.props +++ b/builds/win32/msvc15/FirebirdRelease.props @@ -8,5 +8,8 @@ MultiThreadedDLL /Zo %(AdditionalOptions) + + NDEBUG;%(PreprocessorDefinitions) + \ No newline at end of file diff --git a/builds/win32/msvc15/alice.vcxproj b/builds/win32/msvc15/alice.vcxproj index 44fcd5aa4a..0aed0b4ec8 100644 --- a/builds/win32/msvc15/alice.vcxproj +++ b/builds/win32/msvc15/alice.vcxproj @@ -98,10 +98,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -114,10 +110,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -125,10 +117,6 @@ _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -138,10 +126,6 @@ Disabled _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - diff --git a/builds/win32/msvc15/build_msg.vcxproj b/builds/win32/msvc15/build_msg.vcxproj index b6d8850af2..1451525e6f 100644 --- a/builds/win32/msvc15/build_msg.vcxproj +++ b/builds/win32/msvc15/build_msg.vcxproj @@ -108,10 +108,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -132,10 +128,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -152,10 +144,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -174,10 +162,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/burp.vcxproj b/builds/win32/msvc15/burp.vcxproj index cf6a44ba89..0104a2949b 100644 --- a/builds/win32/msvc15/burp.vcxproj +++ b/builds/win32/msvc15/burp.vcxproj @@ -98,10 +98,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -109,10 +105,6 @@ _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -125,10 +117,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - @@ -138,10 +126,6 @@ Disabled _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - diff --git a/builds/win32/msvc15/codes.vcxproj b/builds/win32/msvc15/codes.vcxproj index 07219856d3..7a4e961251 100644 --- a/builds/win32/msvc15/codes.vcxproj +++ b/builds/win32/msvc15/codes.vcxproj @@ -105,10 +105,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -127,10 +123,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -149,10 +141,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -173,10 +161,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/common.vcxproj b/builds/win32/msvc15/common.vcxproj index e8d6f56042..b69bde6772 100644 --- a/builds/win32/msvc15/common.vcxproj +++ b/builds/win32/msvc15/common.vcxproj @@ -300,10 +300,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) @@ -316,10 +312,6 @@ EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) @@ -335,10 +327,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;%(AdditionalDependencies) @@ -353,10 +341,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;../../../extern/libtommath/lib/$(PlatformName)\$(Configuration)\tommath.lib;../../../extern/libtomcrypt/lib/$(PlatformName)\$(Configuration)\tomcrypt.lib;../../../extern/decNumber/lib/$(PlatformName)\$(Configuration)\decnumber.lib;../../../extern/re2/builds/$(PlatformName)\$(Configuration)\re2.lib;$(TargetDir)\ttmathuint_x86_64_msvc.obj;%(AdditionalDependencies) diff --git a/builds/win32/msvc15/empbuild.vcxproj b/builds/win32/msvc15/empbuild.vcxproj index 0fb11ed15a..b18564a07a 100644 --- a/builds/win32/msvc15/empbuild.vcxproj +++ b/builds/win32/msvc15/empbuild.vcxproj @@ -104,10 +104,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;%(AdditionalDependencies) @@ -134,10 +130,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - Console @@ -153,10 +145,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;fbclient.lib;%(AdditionalDependencies) @@ -178,10 +166,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - odbc32.lib;odbccp32.lib;fbclient.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\fbclient;%(AdditionalLibraryDirectories) diff --git a/builds/win32/msvc15/engine.vcxproj b/builds/win32/msvc15/engine.vcxproj index ea71e3c1f1..f552b6cc89 100644 --- a/builds/win32/msvc15/engine.vcxproj +++ b/builds/win32/msvc15/engine.vcxproj @@ -492,10 +492,6 @@ WIN32;_DEBUG;_WINDOWS;_LIB;SUPERSERVER;DEV_BUILD;NAMESPACE=Vulcan;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) @@ -512,10 +508,6 @@ ../../../src/include;../../../src/include/gen;../../../extern/icu/include;../../../src/vulcan;%(AdditionalIncludeDirectories) WIN32;_DEBUG;_WINDOWS;_LIB;SUPERSERVER;DEV_BUILD;NAMESPACE=Vulcan;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) @@ -533,10 +525,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) @@ -558,10 +546,6 @@ - - NDEBUG;%(PreprocessorDefinitions) - 0x0419 - mpr.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) diff --git a/builds/win32/msvc15/fb2control.vcxproj b/builds/win32/msvc15/fb2control.vcxproj index 52d6df253e..998654a506 100644 --- a/builds/win32/msvc15/fb2control.vcxproj +++ b/builds/win32/msvc15/fb2control.vcxproj @@ -124,8 +124,7 @@ true - _AFXDLL;NDEBUG;%(PreprocessorDefinitions) - 0x040c + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) @@ -155,8 +154,7 @@ true - _AFXDLL;NDEBUG;%(PreprocessorDefinitions) - 0x040c + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) @@ -185,8 +183,7 @@ EditAndContinue - _AFXDLL;_DEBUG;%(PreprocessorDefinitions) - 0x0809 + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) @@ -212,8 +209,7 @@ TRACE;%(UndefinePreprocessorDefinitions) - _AFXDLL;_DEBUG;%(PreprocessorDefinitions) - 0x0809 + _AFXDLL;%(PreprocessorDefinitions) shlwapi.lib;version.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc15/fb_lock_print.vcxproj b/builds/win32/msvc15/fb_lock_print.vcxproj index c8adda160e..c193caa548 100644 --- a/builds/win32/msvc15/fb_lock_print.vcxproj +++ b/builds/win32/msvc15/fb_lock_print.vcxproj @@ -106,10 +106,6 @@ Speed NDEBUG;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -124,10 +120,6 @@ _DEBUG;DEV_BUILD;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -147,10 +139,6 @@ Speed NDEBUG;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -168,10 +156,6 @@ Disabled _DEBUG;DEV_BUILD;_WINDOWS;CLIENT;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/fbguard.vcxproj b/builds/win32/msvc15/fbguard.vcxproj index 14cdd4c1d6..09c3308244 100644 --- a/builds/win32/msvc15/fbguard.vcxproj +++ b/builds/win32/msvc15/fbguard.vcxproj @@ -111,10 +111,6 @@ Speed NDEBUG;WIN32;_WINDOWS;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows @@ -136,10 +132,6 @@ Speed NDEBUG;WIN32;_WINDOWS;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows @@ -160,10 +152,6 @@ _DEBUG;WIN32;_WINDOWS;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows @@ -182,10 +170,6 @@ Disabled _DEBUG;WIN32;_WINDOWS;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Windows diff --git a/builds/win32/msvc15/fbrmclib.vcxproj b/builds/win32/msvc15/fbrmclib.vcxproj index 03ee834ac9..0ce4098d82 100644 --- a/builds/win32/msvc15/fbrmclib.vcxproj +++ b/builds/win32/msvc15/fbrmclib.vcxproj @@ -70,10 +70,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - /MACHINE:I386 /SECTION:.edata,RD %(AdditionalOptions) comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) @@ -98,10 +94,6 @@ NDEBUG;_WINDOWS;_USRDLL;CLIENT;SUPERCLIENT;I386;_X86_=1;WIN32;_X86_;GDS32_EXPORTS;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - /MACHINE:I386 /SECTION:.edata,RD %(AdditionalOptions) comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc15/fbserver.vcxproj b/builds/win32/msvc15/fbserver.vcxproj index 01a823952f..f5cb4f97ff 100644 --- a/builds/win32/msvc15/fbserver.vcxproj +++ b/builds/win32/msvc15/fbserver.vcxproj @@ -116,10 +116,6 @@ NDEBUG;_WINDOWS;SUPERSERVER;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) @@ -144,10 +140,6 @@ NDEBUG;_WINDOWS;SUPERSERVER;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/$(Configuration)/lib;%(AdditionalLibraryDirectories) @@ -171,10 +163,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) @@ -196,10 +184,6 @@ _DEBUG;_WINDOWS;SUPERSERVER;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ../../../extern/icu/$(Platform)/release/lib;%(AdditionalLibraryDirectories) diff --git a/builds/win32/msvc15/fbsvcmgr.vcxproj b/builds/win32/msvc15/fbsvcmgr.vcxproj index 07143e4540..292c68d26b 100644 --- a/builds/win32/msvc15/fbsvcmgr.vcxproj +++ b/builds/win32/msvc15/fbsvcmgr.vcxproj @@ -104,10 +104,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -124,10 +120,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -145,10 +137,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -168,10 +156,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/fbtracemgr.vcxproj b/builds/win32/msvc15/fbtracemgr.vcxproj index 6e2407dc27..9681eb7400 100644 --- a/builds/win32/msvc15/fbtracemgr.vcxproj +++ b/builds/win32/msvc15/fbtracemgr.vcxproj @@ -106,10 +106,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -126,10 +122,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -147,10 +139,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -170,10 +158,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/gbak.vcxproj b/builds/win32/msvc15/gbak.vcxproj index b711b4a70f..4e3bad3de5 100644 --- a/builds/win32/msvc15/gbak.vcxproj +++ b/builds/win32/msvc15/gbak.vcxproj @@ -105,10 +105,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -126,10 +122,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -147,10 +139,6 @@ _DEBUG;DEV_BUILD;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -172,10 +160,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/gfix.vcxproj b/builds/win32/msvc15/gfix.vcxproj index 7eac2c048e..dbf7c2584c 100644 --- a/builds/win32/msvc15/gfix.vcxproj +++ b/builds/win32/msvc15/gfix.vcxproj @@ -105,10 +105,6 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -125,10 +121,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -145,10 +137,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -169,10 +157,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/gpre.vcxproj b/builds/win32/msvc15/gpre.vcxproj index 1161510d0e..8f729f3679 100644 --- a/builds/win32/msvc15/gpre.vcxproj +++ b/builds/win32/msvc15/gpre.vcxproj @@ -108,10 +108,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -133,10 +129,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -153,10 +145,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -175,10 +163,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/gpre_boot.vcxproj b/builds/win32/msvc15/gpre_boot.vcxproj index d1d0d29410..04d707d38b 100644 --- a/builds/win32/msvc15/gpre_boot.vcxproj +++ b/builds/win32/msvc15/gpre_boot.vcxproj @@ -108,10 +108,6 @@ NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -126,10 +122,6 @@ _DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;_WINDOWS;_USRDLL;CLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -149,10 +141,6 @@ Speed NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -171,10 +159,6 @@ _DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;_WINDOWS;_USRDLL;CLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/gpre_common.vcxproj b/builds/win32/msvc15/gpre_common.vcxproj index 13744b5a6b..4c10444d24 100644 --- a/builds/win32/msvc15/gpre_common.vcxproj +++ b/builds/win32/msvc15/gpre_common.vcxproj @@ -104,10 +104,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe @@ -130,10 +126,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe @@ -151,10 +143,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe @@ -174,10 +162,6 @@ GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\bin\$(ProjectName).exe diff --git a/builds/win32/msvc15/gsec.vcxproj b/builds/win32/msvc15/gsec.vcxproj index 96b93e2cc5..5f833a0b96 100644 --- a/builds/win32/msvc15/gsec.vcxproj +++ b/builds/win32/msvc15/gsec.vcxproj @@ -106,10 +106,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -129,10 +125,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -148,10 +140,6 @@ WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -168,10 +156,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/gsplit.vcxproj b/builds/win32/msvc15/gsplit.vcxproj index 9928f861ab..aebab4136b 100644 --- a/builds/win32/msvc15/gsplit.vcxproj +++ b/builds/win32/msvc15/gsplit.vcxproj @@ -106,10 +106,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -127,10 +123,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -148,10 +140,6 @@ _DEBUG;DEV_BUILD;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -173,10 +161,6 @@ NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/gstat.vcxproj b/builds/win32/msvc15/gstat.vcxproj index 154f8289fc..677983308c 100644 --- a/builds/win32/msvc15/gstat.vcxproj +++ b/builds/win32/msvc15/gstat.vcxproj @@ -104,10 +104,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -124,10 +120,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -145,10 +137,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -168,10 +156,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/ib_util.vcxproj b/builds/win32/msvc15/ib_util.vcxproj index 999e0cdb61..554cbf642f 100644 --- a/builds/win32/msvc15/ib_util.vcxproj +++ b/builds/win32/msvc15/ib_util.vcxproj @@ -115,10 +115,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;IB_UTIL_EXPORTS;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def @@ -141,10 +137,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;IB_UTIL_EXPORTS;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def @@ -166,10 +158,6 @@ WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def @@ -189,10 +177,6 @@ Disabled WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\defs\ib_util.def diff --git a/builds/win32/msvc15/instclient.vcxproj b/builds/win32/msvc15/instclient.vcxproj index 526d8635ca..595fb86a7c 100644 --- a/builds/win32/msvc15/instclient.vcxproj +++ b/builds/win32/msvc15/instclient.vcxproj @@ -108,10 +108,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -128,10 +124,6 @@ WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -153,10 +145,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -175,10 +163,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/builds/win32/msvc15/instreg.vcxproj b/builds/win32/msvc15/instreg.vcxproj index 19d0f75c15..75909d0aa8 100644 --- a/builds/win32/msvc15/instreg.vcxproj +++ b/builds/win32/msvc15/instreg.vcxproj @@ -105,10 +105,6 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -126,10 +122,6 @@ Speed WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -148,10 +140,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -173,10 +161,6 @@ Speed WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/builds/win32/msvc15/instsvc.vcxproj b/builds/win32/msvc15/instsvc.vcxproj index b33aeed946..c8b864d2c9 100644 --- a/builds/win32/msvc15/instsvc.vcxproj +++ b/builds/win32/msvc15/instsvc.vcxproj @@ -107,10 +107,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -127,10 +123,6 @@ WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -151,10 +143,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator @@ -173,10 +161,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/builds/win32/msvc15/intl.vcxproj b/builds/win32/msvc15/intl.vcxproj index ee9d89b800..8f6a4c5dcd 100644 --- a/builds/win32/msvc15/intl.vcxproj +++ b/builds/win32/msvc15/intl.vcxproj @@ -120,10 +120,6 @@ NDEBUG;_WINDOWS;_USRDLL;INTL_EXPORTS;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll @@ -148,10 +144,6 @@ NDEBUG;_WINDOWS;_USRDLL;INTL_EXPORTS;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll @@ -175,10 +167,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll @@ -200,10 +188,6 @@ _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\$(ProjectName)\fb$(ProjectName).dll diff --git a/builds/win32/msvc15/intlbuild.vcxproj b/builds/win32/msvc15/intlbuild.vcxproj index cefa52f05b..e82c203efa 100644 --- a/builds/win32/msvc15/intlbuild.vcxproj +++ b/builds/win32/msvc15/intlbuild.vcxproj @@ -103,10 +103,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) Console @@ -128,10 +124,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) Console @@ -147,10 +139,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;fbclient_ms.lib;%(AdditionalDependencies) @@ -169,10 +157,6 @@ ../../../src/include;../../../src/include/gen;../../../src/jrd;%(AdditionalIncludeDirectories) GPRE_FORTRAN;GPRE_PASCAL;GPRE_COBOL;GPRE_ADA;_DEBUG;_CONSOLE;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x0c0a - /MACHINE:I386 %(AdditionalOptions) odbc32.lib;odbccp32.lib;fbclient_ms.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc15/isql.vcxproj b/builds/win32/msvc15/isql.vcxproj index cdc4576f5e..af4b94877b 100644 --- a/builds/win32/msvc15/isql.vcxproj +++ b/builds/win32/msvc15/isql.vcxproj @@ -106,10 +106,6 @@ true EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -127,10 +123,6 @@ NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -148,10 +140,6 @@ _DEBUG;_CONSOLE;SUPERCLIENT;DEV_BUILD;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -173,10 +161,6 @@ NDEBUG;_CONSOLE;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) true - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/nbackup.vcxproj b/builds/win32/msvc15/nbackup.vcxproj index 304a166ad6..f5db2a4f86 100644 --- a/builds/win32/msvc15/nbackup.vcxproj +++ b/builds/win32/msvc15/nbackup.vcxproj @@ -104,10 +104,6 @@ _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -124,10 +120,6 @@ Disabled _DEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -145,10 +137,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) $(OutDir)$(TargetName)$(TargetExt) @@ -169,10 +157,6 @@ Speed NDEBUG;WIN32;_CONSOLE;SUPERCLIENT;CLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/qli.vcxproj b/builds/win32/msvc15/qli.vcxproj index 47ca3ed849..ddc0915102 100644 --- a/builds/win32/msvc15/qli.vcxproj +++ b/builds/win32/msvc15/qli.vcxproj @@ -104,10 +104,6 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -124,10 +120,6 @@ Disabled WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -145,10 +137,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - /MACHINE:I386 %(AdditionalOptions) comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) @@ -170,10 +158,6 @@ Speed WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/builds/win32/msvc15/remote.vcxproj b/builds/win32/msvc15/remote.vcxproj index 4ff3ca3bb4..824f8f8c8b 100644 --- a/builds/win32/msvc15/remote.vcxproj +++ b/builds/win32/msvc15/remote.vcxproj @@ -96,10 +96,6 @@ _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) @@ -112,10 +108,6 @@ Disabled _DEBUG;_LIB;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) @@ -128,10 +120,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) @@ -147,10 +135,6 @@ Speed NDEBUG;_LIB;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ws2_32.lib;%(AdditionalDependencies) diff --git a/builds/win32/msvc15/udf_compat.vcxproj b/builds/win32/msvc15/udf_compat.vcxproj index f64f49812d..d3f63cb210 100644 --- a/builds/win32/msvc15/udf_compat.vcxproj +++ b/builds/win32/msvc15/udf_compat.vcxproj @@ -112,10 +112,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x340a - false @@ -137,10 +133,6 @@ Speed WIN32;NDEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;%(PreprocessorDefinitions) - - NDEBUG;%(PreprocessorDefinitions) - 0x340a - false @@ -161,10 +153,6 @@ WIN32;_DEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x340a - false @@ -183,10 +171,6 @@ Disabled WIN32;_DEBUG;_WINDOWS;_USRDLL;SUPERCLIENT;FBUDF_EXPORTS;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x340a - false diff --git a/builds/win32/msvc15/udr_engine.vcxproj b/builds/win32/msvc15/udr_engine.vcxproj index 3ec3f7fc55..71102206d8 100644 --- a/builds/win32/msvc15/udr_engine.vcxproj +++ b/builds/win32/msvc15/udr_engine.vcxproj @@ -113,10 +113,6 @@ _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll @@ -137,10 +133,6 @@ Disabled _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll @@ -165,10 +157,6 @@ Speed NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll @@ -192,10 +180,6 @@ Speed NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\$(ProjectName).dll diff --git a/builds/win32/msvc15/udrcpp_example.vcxproj b/builds/win32/msvc15/udrcpp_example.vcxproj index 85ebf7b91d..503586897d 100644 --- a/builds/win32/msvc15/udrcpp_example.vcxproj +++ b/builds/win32/msvc15/udrcpp_example.vcxproj @@ -121,10 +121,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll @@ -146,10 +142,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) _DEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;DEV_BUILD;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll @@ -175,10 +167,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll @@ -203,10 +191,6 @@ ..\..\..\src\jrd;%(AdditionalIncludeDirectories) NDEBUG;_WINDOWS;_USRDLL;WINDOWS_ONLY;SUPERCLIENT;WIN32;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) ..\..\..\temp\$(Platform)\$(Configuration)\firebird\plugins\udr\$(ProjectName).dll diff --git a/builds/win32/msvc15/yvalve.vcxproj b/builds/win32/msvc15/yvalve.vcxproj index d1a1c360ac..112d3f9d4b 100644 --- a/builds/win32/msvc15/yvalve.vcxproj +++ b/builds/win32/msvc15/yvalve.vcxproj @@ -175,10 +175,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) @@ -193,10 +189,6 @@ EditAndContinue - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) @@ -214,10 +206,6 @@ Speed WIN32;NDEBUG;_LIB;SUPERSERVER;%(PreprocessorDefinitions) - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) @@ -234,10 +222,6 @@ - - _DEBUG;%(PreprocessorDefinitions) - 0x041d - ..\defs\firebird.def ws2_32.lib;mpr.lib;%(AdditionalDependencies) diff --git a/src/jrd/version.h b/src/jrd/version.h new file mode 100644 index 0000000000..e551e56dbc --- /dev/null +++ b/src/jrd/version.h @@ -0,0 +1,151 @@ +/* + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Vladyslav Khorsun + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2019 Vladyslav Khorsun + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + */ + +#ifndef JRD_VERSION_H +#define JRD_VERSION_H + +#define STRINGIZE_AUX(x) #x +#define STRINGIZE(x) STRINGIZE_AUX(x) + + +#ifdef RC_TARGET_engine13 +#define VER_FILEDESC "Engine plugin" + +#elif defined RC_TARGET_fb_lock_print +#define VER_FILEDESC "Lock Print tool" + +#elif defined RC_TARGET_fbguard +#define VER_FILEDESC "Guardian" + +#elif defined RC_TARGET_fbrmclib +#define VER_FILEDESC "RM/COBOL Helper library" + +#elif defined RC_TARGET_firebird +#define VER_FILEDESC "Server executable" + +#elif defined RC_TARGET_fbsvcmgr +#define VER_FILEDESC "Services Management tool" + +#elif defined RC_TARGET_fbtrace +#define VER_FILEDESC "Trace plugin" + +#elif defined RC_TARGET_fbtracemgr +#define VER_FILEDESC "Trace Management tool" + +#elif defined RC_TARGET_gbak +#define VER_FILEDESC "Gbak tool" + +#elif defined RC_TARGET_gfix +#define VER_FILEDESC "Gfix tool" + +#elif defined RC_TARGET_gpre +#define VER_FILEDESC "Gpre tool" + +#elif defined RC_TARGET_gsec +#define VER_FILEDESC "Gsec tool" + +#elif defined RC_TARGET_gsplit +#define VER_FILEDESC "Gsplit tool" + +#elif defined RC_TARGET_gstat +#define VER_FILEDESC "Gstat tool" + +#elif defined RC_TARGET_ib_util +#define VER_FILEDESC "UDF Helper library" + +#elif defined RC_TARGET_instclient +#define VER_FILEDESC "Install Client tool" + +#elif defined RC_TARGET_instreg +#define VER_FILEDESC "Install Registry tool" + +#elif defined RC_TARGET_instsvc +#define VER_FILEDESC "Install Service tool" + +#elif defined RC_TARGET_fbintl +#define VER_FILEDESC "International Characters support library" + +#elif defined RC_TARGET_isql +#define VER_FILEDESC "Interactive Query tool" + +#elif defined RC_TARGET_legacy_auth +#define VER_FILEDESC "Legacy Auth plugin" + +#elif defined RC_TARGET_legacy_usermanager +#define VER_FILEDESC "Legacy User Manager plugin" + +#elif defined RC_TARGET_nbackup +#define VER_FILEDESC "Physical Backup Management tool" + +#elif defined RC_TARGET_qli +#define VER_FILEDESC "QLI tool" + +#elif defined RC_TARGET_srp +#define VER_FILEDESC "SRP User Manager plugin" + +#elif defined RC_TARGET_udf_compat +#define VER_FILEDESC "UDF compatibility library" + +#elif defined RC_TARGET_udr_engine +#define VER_FILEDESC "User Defined Routines engine" + +#elif defined RC_TARGET_fbclient +#define VER_FILEDESC "Client library" + +#elif defined RC_TARGET_build_msg +#define VER_FILEDESC "Build Message File tool" + +#elif defined RC_TARGET_codes +#define VER_FILEDESC "Generate Error Codes tool" + +#elif defined RC_TARGET_gpre_boot +#define VER_FILEDESC "Bootstrap Gpre tool" + +#elif defined RC_TARGET_udrcpp_example +#define VER_FILEDESC "UDR C++ example" + +#elif defined RC_TARGET_fbudf +#define VER_FILEDESC "UDF library" + +#elif defined RC_TARGET_ib_udf +#define VER_FILEDESC "UDF library" + +#else +#define VER_FILEDESC "SQL Server" + +#endif + + +#ifdef NDEBUG +#define VER_DBG +#else +#define VER_DBG " debug" +#endif + + +#ifdef RC_ARH_x64 +#define VER_ARCH "64-bit" +#else +#define VER_ARCH "32-bit" +#endif + +#endif // JRD_VERSION_H diff --git a/src/jrd/version.rc b/src/jrd/version.rc index ed0f15a77a..caf1a25320 100644 --- a/src/jrd/version.rc +++ b/src/jrd/version.rc @@ -19,6 +19,7 @@ #include #include "../jrd/build_no.h" +#include "../jrd/version.h" VS_VERSION_INFO VERSIONINFO FILEFLAGSMASK VS_FF_PRERELEASE @@ -35,9 +36,10 @@ BEGIN BEGIN VALUE "Comments", "This product created by the Firebird - All Copyright (c) retained by the individual contributors - original code Copyright (c) 2000 Inprise Corporation and predecessors.\0" VALUE "CompanyName", "Firebird Project\0" - VALUE "FileDescription", "Firebird SQL Server\0" + VALUE "FileDescription", "Firebird " VER_FILEDESC ". (" VER_ARCH VER_DBG ")" VALUE "FileVersion", FILE_VER_STRING - VALUE "InternalName", "Firebird\0" + VALUE "InternalName", STRINGIZE(RC_TARGET_NAME) + VALUE "OriginalFilename", STRINGIZE(RC_TARGET_FILENAME) VALUE "LegalCopyright", "All Copyright (c) retained by individual contributors - original code Copyright (c) 2000 Inprise Corporation\0" VALUE "ProductName", "Firebird SQL Server\0" VALUE "ProductVersion", PRODUCT_VER_STRING From c6308c3dc845893a8880cee2f75eb3b9c254efb2 Mon Sep 17 00:00:00 2001 From: hvlad Date: Tue, 24 Dec 2019 19:07:46 +0200 Subject: [PATCH 185/274] Misc --- src/jrd/version.rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jrd/version.rc b/src/jrd/version.rc index caf1a25320..75ff347147 100644 --- a/src/jrd/version.rc +++ b/src/jrd/version.rc @@ -36,7 +36,7 @@ BEGIN BEGIN VALUE "Comments", "This product created by the Firebird - All Copyright (c) retained by the individual contributors - original code Copyright (c) 2000 Inprise Corporation and predecessors.\0" VALUE "CompanyName", "Firebird Project\0" - VALUE "FileDescription", "Firebird " VER_FILEDESC ". (" VER_ARCH VER_DBG ")" + VALUE "FileDescription", "Firebird " VER_FILEDESC " (" VER_ARCH VER_DBG ")" VALUE "FileVersion", FILE_VER_STRING VALUE "InternalName", STRINGIZE(RC_TARGET_NAME) VALUE "OriginalFilename", STRINGIZE(RC_TARGET_FILENAME) From 2c9aec28892ad1d5ea2a4b48cefc87c57c919ba6 Mon Sep 17 00:00:00 2001 From: hvlad Date: Tue, 24 Dec 2019 19:08:16 +0200 Subject: [PATCH 186/274] Description for new ChaCha plugin --- src/jrd/version.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/jrd/version.h b/src/jrd/version.h index e551e56dbc..c7874461b2 100644 --- a/src/jrd/version.h +++ b/src/jrd/version.h @@ -27,7 +27,10 @@ #define STRINGIZE(x) STRINGIZE_AUX(x) -#ifdef RC_TARGET_engine13 +#ifdef RC_TARGET_chacha +#define VER_FILEDESC "Wire Encryption plugin using ChaCha cypher" + +#elif RC_TARGET_engine13 #define VER_FILEDESC "Engine plugin" #elif defined RC_TARGET_fb_lock_print From 1d40a142717e8e123a31771e8c6430e06b6fb8ff Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 25 Dec 2019 00:04:30 +0000 Subject: [PATCH 187/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 7630186e95..ec732d74a6 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1700 + FORMAL BUILD NUMBER:1707 */ -#define PRODUCT_VER_STRING "4.0.0.1700" -#define FILE_VER_STRING "WI-T4.0.0.1700" -#define LICENSE_VER_STRING "WI-T4.0.0.1700" -#define FILE_VER_NUMBER 4, 0, 0, 1700 +#define PRODUCT_VER_STRING "4.0.0.1707" +#define FILE_VER_STRING "WI-T4.0.0.1707" +#define LICENSE_VER_STRING "WI-T4.0.0.1707" +#define FILE_VER_NUMBER 4, 0, 0, 1707 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1700" +#define FB_BUILD_NO "1707" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index e7405fbbb6..66f850096a 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1700 +BuildNum=1707 NowAt=`pwd` cd `dirname $0` From fb8e15bf009de7d4d9346c319860280d057184d3 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Wed, 25 Dec 2019 19:56:39 +0300 Subject: [PATCH 188/274] Enhanced SET BIND OF numeric/decimal as suggested by Mark --- src/common/dsc.h | 8 ++++++++ src/dsql/parse.y | 5 +++++ src/jrd/Coercion.cpp | 14 ++++++++++---- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/common/dsc.h b/src/common/dsc.h index 1cf2e906e3..a29c6a5dda 100644 --- a/src/common/dsc.h +++ b/src/common/dsc.h @@ -218,6 +218,14 @@ typedef struct dsc return isc_blob_text; } + SSHORT getSubType() const + { + if (isBlob() || isExact()) + return dsc_sub_type; + + return 0; + } + void setBlobSubType(SSHORT subType) { if (isBlob()) diff --git a/src/dsql/parse.y b/src/dsql/parse.y index 26119fa93d..1ee02446c2 100644 --- a/src/dsql/parse.y +++ b/src/dsql/parse.y @@ -4649,18 +4649,21 @@ non_charset_simple_type $$->dtype = dtype_int64; $$->length = sizeof(SINT64); + $$->flags |= FLD_has_len; } | integer_keyword { $$ = newNode(); $$->dtype = dtype_long; $$->length = sizeof(SLONG); + $$->flags |= FLD_has_len; } | SMALLINT { $$ = newNode(); $$->dtype = dtype_short; $$->length = sizeof(SSHORT); + $$->flags |= FLD_has_len; } | DATE { @@ -4960,11 +4963,13 @@ numeric_type { $$ = $2; $$->subType = dsc_num_type_numeric; + $$->flags |= FLD_has_sub; } | decimal_keyword prec_scale { $$ = $2; $$->subType = dsc_num_type_decimal; + $$->flags |= FLD_has_sub; if ($$->dtype == dtype_short) { diff --git a/src/jrd/Coercion.cpp b/src/jrd/Coercion.cpp index 90df9697d3..ca7f6acc43 100644 --- a/src/jrd/Coercion.cpp +++ b/src/jrd/Coercion.cpp @@ -144,7 +144,7 @@ bool CoercionRule::match(const dsc* d) const (d->dsc_dtype == fromDsc.dsc_dtype) && ((d->dsc_length == fromDsc.dsc_length) || (!(fromMask & FLD_has_len))) && ((d->getCharSet() == fromDsc.getCharSet()) || (!(fromMask & FLD_has_chset))) && - ((d->getBlobSubType() == fromDsc.getBlobSubType()) || (!(fromMask & FLD_has_sub))) && + ((d->getSubType() == fromDsc.getSubType()) || (!(fromMask & FLD_has_sub))) && ((d->dsc_scale == fromDsc.dsc_scale) || (!(fromMask & FLD_has_scale)))) { found = true; @@ -158,10 +158,16 @@ bool CoercionRule::match(const dsc* d) const case dtype_dec64: case dtype_dec128: if (d->dsc_dtype == dtype_dec64 || d->dsc_dtype == dtype_dec128) - { found = true; - break; - } + break; + + case dtype_short: + case dtype_long: + case dtype_int64: + case dtype_int128: + if (d->isExact() && (fromMask & FLD_has_sub) && (d->dsc_sub_type != dsc_num_type_none)) + found = true; + break; } } From 53b977cf3d33977291373c2468b551f5ab7512d6 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Thu, 26 Dec 2019 00:04:27 +0000 Subject: [PATCH 189/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index ec732d74a6..47d2c8f8d5 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1707 + FORMAL BUILD NUMBER:1708 */ -#define PRODUCT_VER_STRING "4.0.0.1707" -#define FILE_VER_STRING "WI-T4.0.0.1707" -#define LICENSE_VER_STRING "WI-T4.0.0.1707" -#define FILE_VER_NUMBER 4, 0, 0, 1707 +#define PRODUCT_VER_STRING "4.0.0.1708" +#define FILE_VER_STRING "WI-T4.0.0.1708" +#define LICENSE_VER_STRING "WI-T4.0.0.1708" +#define FILE_VER_NUMBER 4, 0, 0, 1708 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1707" +#define FB_BUILD_NO "1708" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 66f850096a..7cffe8ea60 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1707 +BuildNum=1708 NowAt=`pwd` cd `dirname $0` From 816b18e66c1387be8a08531d0986dc5c574b2e5f Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Thu, 26 Dec 2019 15:44:01 +0300 Subject: [PATCH 190/274] Fixed CORE-6212: Authentication plugin on server may get garbage data from client instead empty packet --- src/remote/client/interface.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/remote/client/interface.cpp b/src/remote/client/interface.cpp index bc58969b17..b34791028a 100644 --- a/src/remote/client/interface.cpp +++ b/src/remote/client/interface.cpp @@ -7550,6 +7550,7 @@ static void authReceiveResponse(bool havePacket, ClntAuthBlock& cBlock, rem_port break; } + cBlock.resetDataFromPlugin(); cBlock.storeDataForPlugin(d->cstr_length, d->cstr_address); HANDSHAKE_DEBUG(fprintf(stderr, "Cli: receiveResponse: authenticate(%s)\n", cBlock.plugins.name())); if (cBlock.plugins.plugin()->authenticate(&s, &cBlock) == IAuth::AUTH_FAILED) From 53013ff80f7ee84b6cdcc98e20a02a20ea4f3c7f Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Thu, 26 Dec 2019 19:40:16 +0300 Subject: [PATCH 191/274] Cleanup - remove interbase-times hack with storing length of a string representing some numeric in dsc_sub_type --- src/dsql/ExprNodes.cpp | 22 +++++++++------------- src/dsql/ExprNodes.h | 3 ++- src/dsql/make.cpp | 2 +- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/dsql/ExprNodes.cpp b/src/dsql/ExprNodes.cpp index a8fee2360a..71353a6448 100644 --- a/src/dsql/ExprNodes.cpp +++ b/src/dsql/ExprNodes.cpp @@ -7304,7 +7304,7 @@ static RegisterNode regLiteralNode(blr_literal); LiteralNode::LiteralNode(MemoryPool& pool) : TypedNode(pool), - dsqlStr(NULL) + dsqlStr(NULL), litNumStringLength(0) { litDesc.clear(); } @@ -7444,20 +7444,19 @@ void LiteralNode::genNegZero(DsqlCompilerScratch* dsqlScratch, int prec) dsc desc; desc.dsc_dtype = dtype_double; desc.dsc_scale = 0; - desc.dsc_sub_type = static_cast(s - buf); // Keep length in sub_type which is unused desc.dsc_length = sizeof(double); desc.dsc_address = (UCHAR*) buf; GEN_descriptor(dsqlScratch, &desc, true); - const USHORT len = desc.dsc_sub_type; + const USHORT len = static_cast(s - buf); dsqlScratch->appendUShort(len); if (len) dsqlScratch->appendBytes(desc.dsc_address, len); } // Generate BLR for a constant. -void LiteralNode::genConstant(DsqlCompilerScratch* dsqlScratch, const dsc* desc, bool negateValue) +void LiteralNode::genConstant(DsqlCompilerScratch* dsqlScratch, const dsc* desc, bool negateValue, USHORT numStringLength) { SLONG value; SINT64 i64value; @@ -7527,19 +7526,16 @@ void LiteralNode::genConstant(DsqlCompilerScratch* dsqlScratch, const dsc* desc, GEN_descriptor(dsqlScratch, desc, true); - // Length of string literal - keep it in sub_type which is unused - const USHORT l = desc->dsc_sub_type; - if (negateValue) { - dsqlScratch->appendUShort(l + 1); + dsqlScratch->appendUShort(numStringLength + 1); dsqlScratch->appendUChar('-'); } else - dsqlScratch->appendUShort(l); + dsqlScratch->appendUShort(numStringLength); - if (l) - dsqlScratch->appendBytes(p, l); + if (numStringLength) + dsqlScratch->appendBytes(p, numStringLength); break; } @@ -7748,7 +7744,7 @@ void LiteralNode::genBlr(DsqlCompilerScratch* dsqlScratch) if (litDesc.dsc_dtype == dtype_text) litDesc.dsc_length = dsqlStr->getString().length(); - genConstant(dsqlScratch, &litDesc, false); + genConstant(dsqlScratch, &litDesc, false, litNumStringLength); } void LiteralNode::make(DsqlCompilerScratch* /*dsqlScratch*/, dsc* desc) @@ -8632,7 +8628,7 @@ void NegateNode::genBlr(DsqlCompilerScratch* dsqlScratch) LiteralNode* literal = nodeAs(arg); if (literal && DTYPE_IS_NUMERIC(literal->litDesc.dsc_dtype)) - LiteralNode::genConstant(dsqlScratch, &literal->litDesc, true); + LiteralNode::genConstant(dsqlScratch, &literal->litDesc, true, literal->litNumStringLength); else { dsqlScratch->appendUChar(blr_negate); diff --git a/src/dsql/ExprNodes.h b/src/dsql/ExprNodes.h index 44fd4c4b0b..2a85331308 100644 --- a/src/dsql/ExprNodes.h +++ b/src/dsql/ExprNodes.h @@ -899,7 +899,7 @@ public: explicit LiteralNode(MemoryPool& pool); static DmlNode* parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* csb, const UCHAR blrOp); - static void genConstant(DsqlCompilerScratch* dsqlScratch, const dsc* desc, bool negateValue); + static void genConstant(DsqlCompilerScratch* dsqlScratch, const dsc* desc, bool negateValue, USHORT numStringLength = 0); static void genNegZero(DsqlCompilerScratch* dsqlScratch, int prec); virtual Firebird::string internalPrint(NodePrinter& printer) const; @@ -928,6 +928,7 @@ public: public: const IntlString* dsqlStr; dsc litDesc; + USHORT litNumStringLength; }; diff --git a/src/dsql/make.cpp b/src/dsql/make.cpp index eaa17e8c40..cc03072729 100644 --- a/src/dsql/make.cpp +++ b/src/dsql/make.cpp @@ -205,7 +205,7 @@ ValueExprNode* MAKE_constant(const char* str, dsql_constant_type numeric_flag, S { ERRD_post(Arg::Gds(isc_imp_exc) << Arg::Gds(isc_num_literal)); } - literal->litDesc.dsc_sub_type = static_cast(l); // Keep length in sub_type which is unused + literal->litNumStringLength = static_cast(l); literal->litDesc.dsc_length = numeric_flag == CONSTANT_DOUBLE ? sizeof(double) : numeric_flag == CONSTANT_DECIMAL ? sizeof(Decimal128) : sizeof(Int128); literal->litDesc.dsc_address = (UCHAR*) str; From cd78c265d38baaf6aa5140762ade54b63e92d18c Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Fri, 27 Dec 2019 00:04:26 +0000 Subject: [PATCH 192/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 47d2c8f8d5..9686e1c28a 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1708 + FORMAL BUILD NUMBER:1710 */ -#define PRODUCT_VER_STRING "4.0.0.1708" -#define FILE_VER_STRING "WI-T4.0.0.1708" -#define LICENSE_VER_STRING "WI-T4.0.0.1708" -#define FILE_VER_NUMBER 4, 0, 0, 1708 +#define PRODUCT_VER_STRING "4.0.0.1710" +#define FILE_VER_STRING "WI-T4.0.0.1710" +#define LICENSE_VER_STRING "WI-T4.0.0.1710" +#define FILE_VER_NUMBER 4, 0, 0, 1710 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1708" +#define FB_BUILD_NO "1710" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 7cffe8ea60..c1840bd14d 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1708 +BuildNum=1710 NowAt=`pwd` cd `dirname $0` From a46a3df0fba0dfb37b0c51e1369c279666d5d067 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 27 Dec 2019 15:46:45 +0300 Subject: [PATCH 193/274] Fixup some issues with numeric & decimal datatypes --- doc/sql.extensions/README.set_bind.md | 3 +++ src/dsql/ExprNodes.cpp | 2 +- src/dsql/make.cpp | 2 ++ src/jrd/Coercion.cpp | 22 +++++++++++++--------- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/doc/sql.extensions/README.set_bind.md b/doc/sql.extensions/README.set_bind.md index 40c4497120..ce4da59d23 100644 --- a/doc/sql.extensions/README.set_bind.md +++ b/doc/sql.extensions/README.set_bind.md @@ -24,6 +24,9 @@ When incomplete type definition is used (i.e. `CHAR` instead `CHAR(n)`) in left will take place for all `CHAR` columns, not only default `CHAR(1)`. When incomplete type definiton is used in right side of the statement (TO part) firebird engine will define missing details about that type automatically based on source column. +From this statement POV there is no difference between NUMERIC and DECIMAL datatypes. Changing bind of any NUMERIC +does not affect appropriate underlying integer type. On contrary, changing bind of integer datatype also affects +appropriate NUMERICs. Special `TO` part format `LEGACY` is used when datatype, missing in previous FB version, should be represented in a way, understandable by old client software (may be with some data losses). The following coercions are done for diff --git a/src/dsql/ExprNodes.cpp b/src/dsql/ExprNodes.cpp index 71353a6448..d645e92d94 100644 --- a/src/dsql/ExprNodes.cpp +++ b/src/dsql/ExprNodes.cpp @@ -7449,7 +7449,7 @@ void LiteralNode::genNegZero(DsqlCompilerScratch* dsqlScratch, int prec) GEN_descriptor(dsqlScratch, &desc, true); - const USHORT len = static_cast(s - buf); + const USHORT len = static_cast(s - buf); dsqlScratch->appendUShort(len); if (len) dsqlScratch->appendBytes(desc.dsc_address, len); diff --git a/src/dsql/make.cpp b/src/dsql/make.cpp index cc03072729..be0c469ad5 100644 --- a/src/dsql/make.cpp +++ b/src/dsql/make.cpp @@ -199,6 +199,8 @@ ValueExprNode* MAKE_constant(const char* str, dsql_constant_type numeric_flag, S { literal->litDesc.dsc_dtype = numeric_flag == CONSTANT_DOUBLE ? dtype_double : numeric_flag == CONSTANT_DECIMAL ? dtype_dec128 : dtype_int128; + if (numeric_flag == CONSTANT_NUM128) + literal->litDesc.dsc_sub_type = dsc_num_type_decimal; literal->litDesc.dsc_scale = scale; size_t l = strlen(str); if (l > MAX_SSHORT) diff --git a/src/jrd/Coercion.cpp b/src/jrd/Coercion.cpp index ca7f6acc43..c58b9e2289 100644 --- a/src/jrd/Coercion.cpp +++ b/src/jrd/Coercion.cpp @@ -137,28 +137,25 @@ bool CoercionRule::operator==(const CoercionRule& rule) const bool CoercionRule::match(const dsc* d) const { - bool found = false; - // check for exact match (taking flags into an account) - if ((!found) && - (d->dsc_dtype == fromDsc.dsc_dtype) && + if ((d->dsc_dtype == fromDsc.dsc_dtype) && ((d->dsc_length == fromDsc.dsc_length) || (!(fromMask & FLD_has_len))) && ((d->getCharSet() == fromDsc.getCharSet()) || (!(fromMask & FLD_has_chset))) && ((d->getSubType() == fromDsc.getSubType()) || (!(fromMask & FLD_has_sub))) && ((d->dsc_scale == fromDsc.dsc_scale) || (!(fromMask & FLD_has_scale)))) { - found = true; + return true; } // check for inexact datatype match when FLD_has_len is not set - if ((!found) && (!(fromMask & FLD_has_len))) + if (!(fromMask & FLD_has_len)) { switch(fromDsc.dsc_dtype) { case dtype_dec64: case dtype_dec128: if (d->dsc_dtype == dtype_dec64 || d->dsc_dtype == dtype_dec128) - found = true; + return true; break; case dtype_short: @@ -166,12 +163,19 @@ bool CoercionRule::match(const dsc* d) const case dtype_int64: case dtype_int128: if (d->isExact() && (fromMask & FLD_has_sub) && (d->dsc_sub_type != dsc_num_type_none)) - found = true; + return true; break; } } - return found; + // ignore minor decimal/numeric difference + if (fromDsc.isExact() && (fromMask & FLD_has_sub) && + (fromDsc.dsc_dtype == d->dsc_dtype) && (d->dsc_sub_type != dsc_num_type_none)) + { + return true; + } + + return false; } bool CoercionRule::coerce(dsc* d) const From 808688a2af0ec4952c44eeb45f4a287388783d86 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 27 Dec 2019 20:17:04 +0300 Subject: [PATCH 194/274] Fixed CORE-6208: Grant lost in security.db after backup/restore cycle --- src/burp/backup.epp | 63 +++++++++++++++++++++++++++++++++++ src/burp/burp.h | 11 +++++-- src/burp/restore.epp | 71 ++++++++++++++++++++++++++++++++++++++++ src/msgs/facilities2.sql | 2 +- src/msgs/messages2.sql | 5 +++ 5 files changed, 149 insertions(+), 3 deletions(-) diff --git a/src/burp/backup.epp b/src/burp/backup.epp index b31f1383ed..68a4a233c9 100644 --- a/src/burp/backup.epp +++ b/src/burp/backup.epp @@ -133,6 +133,7 @@ void write_global_fields(); void write_generators(); void write_sql_roles(); void write_mapping(); +void write_db_creators(); void write_packages(); void write_procedures(); void write_procedure_prms(const GDS_NAME, const GDS_NAME); @@ -400,6 +401,11 @@ int BACKUP_backup(const TEXT* dbb_file, const TEXT* file_name) BURP_verbose(296); // msg 296 writing mapping write_mapping(); + + // Write database creators + BURP_verbose(391); + // msg 391 writing database creators + write_db_creators(); } // Finish up @@ -4110,6 +4116,63 @@ void write_mapping() } +void write_db_creators() +{ +/************************************** + * + * w r i t e _ d b _ c r e a t o r s + * + ************************************** + * + * Functional description + * write a record in the burp file for + * each grant to create database. + * + **************************************/ + Firebird::IRequest* req_handle = nullptr; + TEXT temp[GDS_NAME_LEN]; + + BurpGlobals* tdgbl = BurpGlobals::getSpecific(); + + if (tdgbl->runtimeODS >= DB_VERSION_DDL12) + { + FOR (REQUEST_HANDLE req_handle) + C IN RDB$DB_CREATORS + + bool fl = true; + + if (!C.RDB$USER_TYPE.NULL) + { + put(tdgbl, rec_db_creator); + fl = false; + put_int32(att_dbc_type, C.RDB$USER_TYPE); + } + + if (!C.RDB$USER.NULL) + { + if (fl) + put(tdgbl, rec_db_creator); + fl = false; + const SSHORT l = PUT_TEXT(att_dbc_user, C.RDB$USER); + + MISC_terminate (C.RDB$USER, temp, l, sizeof(temp)); + BURP_verbose (392, temp); + // msg 392 writing db creator %s + } + + if (!fl) + put(tdgbl, att_end); + + END_FOR; + ON_ERROR + general_on_error(); + END_ERROR; + } + + MISC_release_request_silent(req_handle); +} + + void write_triggers() { /************************************** diff --git a/src/burp/burp.h b/src/burp/burp.h index c4eb6e18b9..aeac8df987 100644 --- a/src/burp/burp.h +++ b/src/burp/burp.h @@ -114,7 +114,8 @@ enum rec_type { rec_collation, // Collations rec_sql_roles, // SQL roles rec_mapping, // Mapping of security names - rec_package // Package + rec_package, // Package + rec_db_creator // Database creator }; @@ -625,7 +626,11 @@ enum att_type { att_package_security_class, att_package_owner_name, att_package_description, - att_package_sql_security + att_package_sql_security, + + // Database creators + att_dbc_user = SERIES, + att_dbc_type }; @@ -1098,6 +1103,7 @@ public: Firebird::IRequest* handles_get_security_class_req_handle1; Firebird::IRequest* handles_get_sql_roles_req_handle1; Firebird::IRequest* handles_get_mapping_req_handle1; + Firebird::IRequest* handles_db_creators_req_handle1; Firebird::IRequest* handles_get_trigger_message_req_handle1; Firebird::IRequest* handles_get_trigger_message_req_handle2; Firebird::IRequest* handles_get_trigger_old_req_handle1; @@ -1154,6 +1160,7 @@ public: ULONG verboseInterval; // How many records should be backed up or restored before we show this message bool flag_on_line; // indicates whether we will bring the database on-line bool firstMap; // this is the first time we entered get_mapping() + bool firstDbc; // this is the first time we entered get_db_creators() bool stdIoMode; // stdin or stdout is used as backup file Firebird::AutoPtr skipDataMatcher; Firebird::AutoPtr includeDataMatcher; diff --git a/src/burp/restore.epp b/src/burp/restore.epp index cf2dc9bb42..9493ae7fa8 100644 --- a/src/burp/restore.epp +++ b/src/burp/restore.epp @@ -147,6 +147,7 @@ bool get_relation(BurpGlobals* tdgbl); bool get_relation_data(BurpGlobals* tdgbl); bool get_sql_roles(BurpGlobals* tdgbl); bool get_mapping(BurpGlobals* tdgbl); +bool get_db_creator(BurpGlobals* tdgbl); bool get_security_class(BurpGlobals* tdgbl); void get_source_blob(BurpGlobals* tdgbl, ISC_QUAD&, bool); USHORT get_text(BurpGlobals* tdgbl, TEXT*, ULONG); @@ -8905,6 +8906,70 @@ bool get_mapping(BurpGlobals* tdgbl) return true; } +bool get_db_creator(BurpGlobals* tdgbl) +{ +/************************************** + * + * g e t _ d b _ c r e a t o r + * + ************************************** + * + * Functional description + * Restore database creators + * + **************************************/ + att_type attribute; + scan_attr_t scan_next_attr; + TEXT temp[GDS_NAME_LEN]; + SSHORT l; + Firebird::string role; + + if (tdgbl->runtimeODS >= DB_VERSION_DDL12) + { + STORE (REQUEST_HANDLE tdgbl->handles_db_creators_req_handle1) + C IN RDB$DB_CREATORS + + C.RDB$USER.NULL = TRUE; + C.RDB$USER_TYPE.NULL = TRUE; + + skip_init(&scan_next_attr); + while (skip_scan(&scan_next_attr), get_attribute(&attribute, tdgbl) != att_end) + { + switch (attribute) + { + case att_dbc_user: + C.RDB$USER.NULL = FALSE; + GET_TEXT(C.RDB$USER); + if (tdgbl->firstDbc) + { + tdgbl->firstDbc = false; + BURP_verbose(394); + // msg 394 restoring database creators + } + BURP_verbose (393, C.RDB$USER); + break; + + case att_dbc_type: + C.RDB$USER_TYPE.NULL = FALSE; + C.RDB$USER_TYPE = (USHORT) get_int32(tdgbl); + break; + + default: + // msg 395 database creator + bad_attribute(scan_next_attr, attribute, 395); + break; + } + } + + END_STORE; + ON_ERROR + general_on_error (); + END_ERROR; + } + + return true; +} + bool is_ascii_name (const TEXT *name, const SSHORT len) { /************************************** @@ -10921,6 +10986,12 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file flag = true; break; + case rec_db_creator: + if (!get_db_creator(tdgbl)) + return false; + flag = true; + break; + default: BURP_error(43, true, SafeArg() << record); // msg 43 don't recognize record type %ld diff --git a/src/msgs/facilities2.sql b/src/msgs/facilities2.sql index 19516660e7..784e64b79a 100644 --- a/src/msgs/facilities2.sql +++ b/src/msgs/facilities2.sql @@ -9,7 +9,7 @@ set bulk_insert INSERT INTO FACILITIES (LAST_CHANGE, FACILITY, FAC_CODE, MAX_NUM ('2018-06-22 11:46:00', 'DYN', 8, 309) ('1996-11-07 13:39:40', 'INSTALL', 10, 1) ('1996-11-07 13:38:41', 'TEST', 11, 4) -('2018-04-26 20:40:00', 'GBAK', 12, 391) +('2019-12-27 20:10:00', 'GBAK', 12, 396) ('2019-04-13 21:10:00', 'SQLERR', 13, 1047) ('1996-11-07 13:38:42', 'SQLWARN', 14, 613) ('2018-02-27 14:50:31', 'JRD_BUGCHK', 15, 308) diff --git a/src/msgs/messages2.sql b/src/msgs/messages2.sql index f2f612197f..0aee80450b 100644 --- a/src/msgs/messages2.sql +++ b/src/msgs/messages2.sql @@ -2553,6 +2553,11 @@ ERROR: Backup incomplete', NULL, NULL); (NULL, 'burp_usage', 'burp.c', NULL, 12, 388, NULL, ' @1INCLUDE(_DATA) backup data of table(s)', NULL, NULL); (NULL, NULL, 'burp.cpp', NULL, 12, 389, NULL, 'missing regular expression to include tables', NULL, NULL); (NULL, NULL, 'burp.cpp', NULL, 12, 390, NULL, 'regular expression to include tables was already set', NULL, NULL); +(NULL, 'BACKUP_backup', 'backup.epp', NULL, 12, 391, NULL, 'writing database create grants', NULL, NULL); +(NULL, 'write_db_creators', 'backup.epp', NULL, 12, 392, NULL, ' database create grant for @1', NULL, NULL); +(NULL, 'get_db_creators', 'restore.epp', NULL, 12, 393, NULL, ' restoring database create grant for @1', NULL, NULL); +(NULL, 'get_db_creators', 'restore.epp', NULL, 12, 394, NULL, 'restoring database create grants', NULL, NULL); +(NULL, 'get_db_creators', 'restore.epp', NULL, 12, 395, NULL, 'database create grant', NULL, NULL); -- SQLERR (NULL, NULL, NULL, NULL, 13, 1, NULL, 'Firebird error', NULL, NULL); (NULL, NULL, NULL, NULL, 13, 74, NULL, 'Rollback not performed', NULL, NULL); From 71496ddad3b73194da678c4dc8b64112d11e997b Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 28 Dec 2019 00:04:34 +0000 Subject: [PATCH 195/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 9686e1c28a..3b7845957d 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1710 + FORMAL BUILD NUMBER:1712 */ -#define PRODUCT_VER_STRING "4.0.0.1710" -#define FILE_VER_STRING "WI-T4.0.0.1710" -#define LICENSE_VER_STRING "WI-T4.0.0.1710" -#define FILE_VER_NUMBER 4, 0, 0, 1710 +#define PRODUCT_VER_STRING "4.0.0.1712" +#define FILE_VER_STRING "WI-T4.0.0.1712" +#define LICENSE_VER_STRING "WI-T4.0.0.1712" +#define FILE_VER_NUMBER 4, 0, 0, 1712 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1710" +#define FB_BUILD_NO "1712" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index c1840bd14d..8ac88b1efb 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1710 +BuildNum=1712 NowAt=`pwd` cd `dirname $0` From 57b0f8ab210024866d10f77219f76dc375753d49 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sun, 29 Dec 2019 15:13:35 -0300 Subject: [PATCH 196/274] Complement fix for CORE-6211: This change makes gpre_boot and gpre boot generate identical files. Hence it fixes second Linux build (after initial one, when epp files are changed) and Windows build (that always re-process epp files with gpre even in the initial build). --- src/gpre/cmp.cpp | 4 ++-- src/gpre/jrdmet.cpp | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/gpre/cmp.cpp b/src/gpre/cmp.cpp index a644c9ac03..e8bca853c7 100644 --- a/src/gpre/cmp.cpp +++ b/src/gpre/cmp.cpp @@ -756,7 +756,7 @@ static void cmp_field( gpre_req* request, const gpre_fld* field, switch (field->fld_dtype) { case dtype_cstring: - if (!(field->fld_flags & FLD_charset) && field->fld_ttype && field->fld_ttype != CS_BINARY) + if (!(field->fld_flags & FLD_charset) && field->fld_ttype >= dsc_text_type_metadata) { request->add_byte(blr_cstring); request->add_word(field->fld_length); @@ -772,7 +772,7 @@ static void cmp_field( gpre_req* request, const gpre_fld* field, break; case dtype_text: - if (!(field->fld_flags & FLD_charset) && field->fld_ttype) + if (!(field->fld_flags & FLD_charset) && field->fld_ttype >= dsc_text_type_metadata) { request->add_byte(blr_text); request->add_word(field->fld_length); diff --git a/src/gpre/jrdmet.cpp b/src/gpre/jrdmet.cpp index 74162bf3df..d11b6df001 100644 --- a/src/gpre/jrdmet.cpp +++ b/src/gpre/jrdmet.cpp @@ -78,15 +78,20 @@ void JRDMET_init( gpre_dbb* db) field->fld_sub_type = gfield->gfld_sub_type; if (field->fld_dtype == dtype_varying || field->fld_dtype == dtype_text) { - field->fld_dtype = dtype_cstring; + if (gfield->gfld_sub_type == dsc_text_type_fixed) + field->fld_dtype = dtype_text; + else + { + field->fld_dtype = dtype_cstring; + ++field->fld_length; + } + field->fld_flags |= FLD_text; if (gfield->gfld_sub_type == dsc_text_type_metadata) { if (gpreGlob.sw_language == lang_internal) field->fld_flags |= FLD_charset; - else - field->fld_length *= 4; field->fld_charset_id = CS_METADATA; field->fld_collate_id = COLLATE_NONE; @@ -97,12 +102,10 @@ void JRDMET_init( gpre_dbb* db) if (gpreGlob.sw_language == lang_internal) field->fld_flags |= FLD_charset; - field->fld_charset_id = CS_NONE; + field->fld_charset_id = gfield->gfld_sub_type == dsc_text_type_fixed ? CS_BINARY : CS_NONE; field->fld_collate_id = COLLATE_NONE; field->fld_ttype = gfield->gfld_sub_type == dsc_text_type_fixed ? ttype_binary : ttype_none; } - - ++field->fld_length; } else if (field->fld_dtype == dtype_blob) { From f784e476c4dcc7280932b2e383d6b5469508df60 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Tue, 31 Dec 2019 00:04:23 +0000 Subject: [PATCH 197/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 3b7845957d..48cf6f005e 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1712 + FORMAL BUILD NUMBER:1713 */ -#define PRODUCT_VER_STRING "4.0.0.1712" -#define FILE_VER_STRING "WI-T4.0.0.1712" -#define LICENSE_VER_STRING "WI-T4.0.0.1712" -#define FILE_VER_NUMBER 4, 0, 0, 1712 +#define PRODUCT_VER_STRING "4.0.0.1713" +#define FILE_VER_STRING "WI-T4.0.0.1713" +#define LICENSE_VER_STRING "WI-T4.0.0.1713" +#define FILE_VER_NUMBER 4, 0, 0, 1713 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1712" +#define FB_BUILD_NO "1713" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 8ac88b1efb..e9bff5e56a 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1712 +BuildNum=1713 NowAt=`pwd` cd `dirname $0` From 04fdeabbb426b8029a60962ffae886a4981d1289 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Sat, 4 Jan 2020 12:31:30 +0300 Subject: [PATCH 198/274] Cleanup --- src/yvalve/why.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/yvalve/why.cpp b/src/yvalve/why.cpp index c67527d564..1df43ca04b 100644 --- a/src/yvalve/why.cpp +++ b/src/yvalve/why.cpp @@ -27,8 +27,6 @@ * */ -//#define FB_COMPILING_WHY_CPP - #include "firebird.h" #include "firebird/Interface.h" #include "memory_routines.h" From 44cc992440405d5425a2183578b3679f48c07f0b Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sun, 5 Jan 2020 00:04:25 +0000 Subject: [PATCH 199/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 48cf6f005e..e1173f0d80 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1713 + FORMAL BUILD NUMBER:1714 */ -#define PRODUCT_VER_STRING "4.0.0.1713" -#define FILE_VER_STRING "WI-T4.0.0.1713" -#define LICENSE_VER_STRING "WI-T4.0.0.1713" -#define FILE_VER_NUMBER 4, 0, 0, 1713 +#define PRODUCT_VER_STRING "4.0.0.1714" +#define FILE_VER_STRING "WI-T4.0.0.1714" +#define LICENSE_VER_STRING "WI-T4.0.0.1714" +#define FILE_VER_NUMBER 4, 0, 0, 1714 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1713" +#define FB_BUILD_NO "1714" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index e9bff5e56a..4c42c7ce3b 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1713 +BuildNum=1714 NowAt=`pwd` cd `dirname $0` From 0453bbec9ec71ff905d8ebc5e1591c77f28f2da8 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Thu, 9 Jan 2020 18:46:37 +0300 Subject: [PATCH 200/274] Implemented CORE-6220: Enable delivery of known to the client key to any connection to the server --- src/common/classes/GetPlugins.h | 86 +++++++++++++++++++++++---------- src/remote/client/interface.cpp | 65 ++++++++++++++++++++++++- src/remote/remote.h | 24 +++++++++ 3 files changed, 148 insertions(+), 27 deletions(-) diff --git a/src/common/classes/GetPlugins.h b/src/common/classes/GetPlugins.h index 8ee4333561..a6cb430732 100644 --- a/src/common/classes/GetPlugins.h +++ b/src/common/classes/GetPlugins.h @@ -33,6 +33,7 @@ #include "../common/classes/auto.h" #include "../common/config/config.h" #include "../common/StatusHolder.h" +#include "../common/classes/fb_string.h" namespace Firebird { @@ -41,28 +42,32 @@ template class GetPlugins { public: - GetPlugins(unsigned int interfaceType, const char* namesList = NULL) - : masterInterface(), pluginInterface(), - pluginSet(NULL), currentPlugin(NULL), - ls(*getDefaultMemoryPool()), status(&ls) + GetPlugins(unsigned int iType, const char* namesList = NULL) + : pluginList(*getDefaultMemoryPool()), + masterInterface(), pluginInterface(), + currentPlugin(NULL), + ls(*getDefaultMemoryPool()), status(&ls), + interfaceType(iType) { + pluginList = namesList ? namesList : Config::getDefaultConfig()->getPlugins(interfaceType); pluginSet.assignRefNoIncr(pluginInterface->getPlugins(&status, interfaceType, - (namesList ? namesList : Config::getDefaultConfig()->getPlugins(interfaceType)), - NULL)); + pluginList.c_str(), NULL)); check(&status); getPlugin(); } - GetPlugins(unsigned int interfaceType, - const Config* knownConfig, const char* namesList = NULL) - : masterInterface(), pluginInterface(), - pluginSet(NULL), currentPlugin(NULL), - ls(*getDefaultMemoryPool()), status(&ls) + GetPlugins(unsigned int iType, + const Config* conf, const char* namesList = NULL) + : pluginList(*getDefaultMemoryPool()), + masterInterface(), pluginInterface(), + knownConfig(conf), currentPlugin(NULL), + ls(*getDefaultMemoryPool()), status(&ls), + interfaceType(iType) { + pluginList = namesList ? namesList : knownConfig->getPlugins(interfaceType); pluginSet.assignRefNoIncr(pluginInterface->getPlugins(&status, interfaceType, - (namesList ? namesList : knownConfig->getPlugins(interfaceType)), - FB_NEW FirebirdConf(knownConfig))); + pluginList.c_str(), FB_NEW FirebirdConf(knownConfig))); check(&status); getPlugin(); @@ -97,8 +102,7 @@ public: { if (hasData()) { - pluginInterface->releasePlugin(currentPlugin); - currentPlugin = NULL; + removePlugin(); pluginSet->next(&status); check(&status); @@ -108,39 +112,69 @@ public: void set(const char* newName) { - if (hasData()) - { - pluginInterface->releasePlugin(currentPlugin); - currentPlugin = NULL; - } + removePlugin(); - pluginSet->set(&status, newName); + pluginList = newName; + pluginSet->set(&status, pluginList.c_str()); check(&status); + + getPlugin(); + } + + void set(const Config* conf) + { + removePlugin(); + + knownConfig = conf; + pluginList = knownConfig->getPlugins(interfaceType); + pluginSet.assignRefNoIncr(pluginInterface->getPlugins(&status, interfaceType, + pluginList.c_str(), FB_NEW FirebirdConf(knownConfig))); + check(&status); + + getPlugin(); + } + + void rewind() + { + removePlugin(); + + pluginSet.assignRefNoIncr(pluginInterface->getPlugins(&status, interfaceType, + pluginList.c_str(), knownConfig.hasData() ? FB_NEW FirebirdConf(knownConfig) : NULL)); + check(&status); + getPlugin(); } ~GetPlugins() { - if (hasData()) - { - pluginInterface->releasePlugin(currentPlugin); - currentPlugin = NULL; - } + removePlugin(); } private: + PathName pluginList; MasterInterfacePtr masterInterface; PluginManagerInterfacePtr pluginInterface; + RefPtr knownConfig; RefPtr pluginSet; P* currentPlugin; LocalStatus ls; CheckStatusWrapper status; + unsigned interfaceType; void getPlugin() { currentPlugin = (P*) pluginSet->getPlugin(&status); check(&status); } + + void removePlugin() + { + if (hasData()) + { + pluginInterface->releasePlugin(currentPlugin); + currentPlugin = NULL; + } + } }; // template required to use AutoPtr for plugins diff --git a/src/remote/client/interface.cpp b/src/remote/client/interface.cpp index b34791028a..18fa388fbb 100644 --- a/src/remote/client/interface.cpp +++ b/src/remote/client/interface.cpp @@ -7615,6 +7615,7 @@ static void init(CheckStatusWrapper* status, ClntAuthBlock& cBlock, rem_port* po authFillParametersBlock(cBlock, dpb, ps, port); port->port_client_crypt_callback = cryptCallback; + cBlock.createCryptCallback(&port->port_client_crypt_callback); // Make attach packet P_ATCH* attach = &packet->p_atch; @@ -8735,7 +8736,8 @@ ClntAuthBlock::ClntAuthBlock(const Firebird::PathName* fileName, Firebird::Clump cliUserName(getPool()), cliPassword(getPool()), cliOrigUserName(getPool()), dataForPlugin(getPool()), dataFromPlugin(getPool()), cryptKeys(getPool()), dpbConfig(getPool()), dpbPlugins(getPool()), - plugins(IPluginManager::TYPE_AUTH_CLIENT), authComplete(false), firstTime(true) + plugins(IPluginManager::TYPE_AUTH_CLIENT), authComplete(false), firstTime(true), + createdInterface(nullptr) { if (dpb && tags) { @@ -8985,3 +8987,64 @@ void ClntAuthBlock::releaseKeys(unsigned from) delete cryptKeys[from++]; } } + +void ClntAuthBlock::createCryptCallback(Firebird::ICryptKeyCallback** callback) +{ + if (*callback) + return; + + *callback = clientCrypt.create(clntConfig); + if (*callback) + createdInterface = callback; +} + +Firebird::ICryptKeyCallback* ClntAuthBlock::ClientCrypt::create(const Config* conf) +{ + pluginItr.set(conf); + + return pluginItr.hasData() ? this : nullptr; +} + +unsigned ClntAuthBlock::ClientCrypt::callback(unsigned dlen, const void* data, unsigned blen, void* buffer) +{ + HANDSHAKE_DEBUG(fprintf(stderr, "dlen=%d blen=%d\n", dlen, blen)); + + int loop = 0; + while (loop < 2) + { + for (; pluginItr.hasData(); pluginItr.next()) + { + if (!currentIface) + { + LocalStatus ls; + CheckStatusWrapper st(&ls); + + HANDSHAKE_DEBUG(fprintf(stderr, "Try plugin %s\n", pluginItr.name())); + currentIface = pluginItr.plugin()->chainHandle(&st); + // if plugin does not support chaining - silently ignore it + check(&st, isc_wish_list); + HANDSHAKE_DEBUG(fprintf(stderr, "Use plugin %s, ptr=%p\n", pluginItr.name(), currentIface)); + } + + // if we have an iface - try it + if (currentIface) + { + unsigned retlen = currentIface->callback(dlen, data, blen, buffer); + HANDSHAKE_DEBUG(fprintf(stderr, "Iface %p returned %d\n", currentIface, retlen)); + if (retlen) + return retlen; + } + + // no success with iface - clear it + // appropriate data structures to be released by plugin cleanup code + currentIface = nullptr; + } + + ++loop; + // prepare iterator for next use + pluginItr.rewind(); + } + + // no luck with suggested data + return 0; +} diff --git a/src/remote/remote.h b/src/remote/remote.h index 9346078ceb..6e575106fb 100644 --- a/src/remote/remote.h +++ b/src/remote/remote.h @@ -837,6 +837,26 @@ private: Firebird::AutoPtr remAuthBlock; //Authentication block if present unsigned nextKey; // First key to be analyzed + class ClientCrypt FB_FINAL : + public Firebird::VersionedIface > + { + public: + ClientCrypt() + : pluginItr(Firebird::IPluginManager::TYPE_KEY_HOLDER, "NoDefault"), currentIface(nullptr) + { } + + Firebird::ICryptKeyCallback* create(const Config* conf); + + // Firebird::IClientBlock implementation + unsigned callback(unsigned dataLength, const void* data, unsigned bufferLength, void* buffer); + + private: + Firebird::GetPlugins pluginItr; + Firebird::ICryptKeyCallback* currentIface; + }; + ClientCrypt clientCrypt; + Firebird::ICryptKeyCallback** createdInterface; + public: AuthClientPlugins plugins; bool authComplete; // Set as response from client that authentication accepted @@ -848,6 +868,9 @@ public: ~ClntAuthBlock() { releaseKeys(0); + + if (createdInterface) + *createdInterface = nullptr; } void storeDataForPlugin(unsigned int length, const unsigned char* data); @@ -863,6 +886,7 @@ public: void tryNewKeys(rem_port*); void releaseKeys(unsigned from); Firebird::RefPtr* getConfig(); + void createCryptCallback(Firebird::ICryptKeyCallback** callback); // Firebird::IClientBlock implementation int release(); From c575383595a030d55c03dd368c5ea649933d5671 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Fri, 10 Jan 2020 00:04:51 +0000 Subject: [PATCH 201/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index e1173f0d80..532c36683a 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1714 + FORMAL BUILD NUMBER:1715 */ -#define PRODUCT_VER_STRING "4.0.0.1714" -#define FILE_VER_STRING "WI-T4.0.0.1714" -#define LICENSE_VER_STRING "WI-T4.0.0.1714" -#define FILE_VER_NUMBER 4, 0, 0, 1714 +#define PRODUCT_VER_STRING "4.0.0.1715" +#define FILE_VER_STRING "WI-T4.0.0.1715" +#define LICENSE_VER_STRING "WI-T4.0.0.1715" +#define FILE_VER_NUMBER 4, 0, 0, 1715 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1714" +#define FB_BUILD_NO "1715" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 4c42c7ce3b..1f6cbcf0ce 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1714 +BuildNum=1715 NowAt=`pwd` cd `dirname $0` From 908c660de5448485e13330f93641241b2cfc709d Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 10 Jan 2020 13:59:12 +0300 Subject: [PATCH 202/274] Fixed CORE-6221: Incorrect (throw-based) allocFunc for zlib --- src/common/classes/zip.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/common/classes/zip.cpp b/src/common/classes/zip.cpp index 24cd1824b6..e62b3b291b 100644 --- a/src/common/classes/zip.cpp +++ b/src/common/classes/zip.cpp @@ -56,11 +56,16 @@ void ZLib::symbols() #undef FB_ZSYMB } -// Firebird::InitInstance zlib; - void* ZLib::allocFunc(void*, uInt items, uInt size) { - return MemoryPool::globalAlloc(items * size ALLOC_ARGS); + try + { + return MemoryPool::globalAlloc(items * size ALLOC_ARGS); + } + catch (const Exception&) + { + return nullptr; + } } void ZLib::freeFunc(void*, void* address) From c6ffc03e1f1d8a920e8df9bbbf077771ce822cf8 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 10 Jan 2020 14:09:48 +0300 Subject: [PATCH 203/274] Fixed CORE-6217: Wrong work with pointer: delete ptr; ptr=new ; --- src/remote/inet.cpp | 1 + src/remote/os/win32/wnet.cpp | 1 + src/remote/os/win32/xnet.cpp | 1 + 3 files changed, 3 insertions(+) diff --git a/src/remote/inet.cpp b/src/remote/inet.cpp index a1bb0ce575..b3ad39729e 100644 --- a/src/remote/inet.cpp +++ b/src/remote/inet.cpp @@ -836,6 +836,7 @@ rem_port* INET_connect(const TEXT* name, if (host.hasData()) { delete port->port_connection; + port->port_connection = nullptr; port->port_connection = REMOTE_make_string(host.c_str()); } else { diff --git a/src/remote/os/win32/wnet.cpp b/src/remote/os/win32/wnet.cpp index b28e2d4e66..b551112bd9 100644 --- a/src/remote/os/win32/wnet.cpp +++ b/src/remote/os/win32/wnet.cpp @@ -538,6 +538,7 @@ static rem_port* alloc_port( rem_port* parent) if (parent) { delete port->port_connection; + port->port_connection = nullptr; port->port_connection = REMOTE_make_string(parent->port_connection->str_data); port->linkParent(parent); diff --git a/src/remote/os/win32/xnet.cpp b/src/remote/os/win32/xnet.cpp index 2c60807df1..7385d0c09e 100644 --- a/src/remote/os/win32/xnet.cpp +++ b/src/remote/os/win32/xnet.cpp @@ -708,6 +708,7 @@ static rem_port* alloc_port(rem_port* parent, if (parent) { delete port->port_connection; + port->port_connection = nullptr; port->port_connection = REMOTE_make_string(parent->port_connection->str_data); port->linkParent(parent); From 43574d1588c4032739ee2aa863c0bef5f687c35a Mon Sep 17 00:00:00 2001 From: Roman Simakov Date: Mon, 2 Dec 2019 12:02:38 +0300 Subject: [PATCH 204/274] Removed TDBB_trusted_ddl flag which switched off all SCL_checks globally. Also removed SCL_checks from VIO_store, VIO_modify, VIO_erase which were switched off by TDBB_trudted_ddl after checkPermission in DdlNode. Other ways to run VIO_xxx are prohibited and users cannot modify system tables directly. Now error includes an object name when user does not have the privilege CREATE for it. --- lang_helpers/gds_codes.ftn | 2 + lang_helpers/gds_codes.pas | 2 + src/dsql/DdlNodes.epp | 2 - src/dsql/Nodes.h | 8 +- src/dsql/PackageNodes.epp | 20 ++--- src/include/gen/codetext.h | 1 + src/include/gen/iberror.h | 6 +- src/include/gen/msgs.h | 1 + src/include/gen/sql_code.h | 1 + src/include/gen/sql_state.h | 1 + src/jrd/jrd.h | 1 - src/jrd/scl.epp | 17 ++-- src/jrd/vio.cpp | 156 +++++++----------------------------- src/msgs/facilities2.sql | 2 +- src/msgs/messages2.sql | 1 + src/msgs/system_errors2.sql | 1 + 16 files changed, 60 insertions(+), 162 deletions(-) diff --git a/lang_helpers/gds_codes.ftn b/lang_helpers/gds_codes.ftn index 1c79f7442b..4cc67858fd 100644 --- a/lang_helpers/gds_codes.ftn +++ b/lang_helpers/gds_codes.ftn @@ -1938,6 +1938,8 @@ C -- PARAMETER (GDS__cannot_update_old_blob = 335545262) INTEGER*4 GDS__cannot_read_new_blob PARAMETER (GDS__cannot_read_new_blob = 335545263) + INTEGER*4 GDS__dyn_no_create_priv + PARAMETER (GDS__dyn_no_create_priv = 335545264) INTEGER*4 GDS__gfix_db_name PARAMETER (GDS__gfix_db_name = 335740929) INTEGER*4 GDS__gfix_invalid_sw diff --git a/lang_helpers/gds_codes.pas b/lang_helpers/gds_codes.pas index bc25a11a9a..e7521d3cb8 100644 --- a/lang_helpers/gds_codes.pas +++ b/lang_helpers/gds_codes.pas @@ -1933,6 +1933,8 @@ const gds_cannot_update_old_blob = 335545262; isc_cannot_read_new_blob = 335545263; gds_cannot_read_new_blob = 335545263; + isc_dyn_no_create_priv = 335545264; + gds_dyn_no_create_priv = 335545264; isc_gfix_db_name = 335740929; gds_gfix_db_name = 335740929; isc_gfix_invalid_sw = 335740930; diff --git a/src/dsql/DdlNodes.epp b/src/dsql/DdlNodes.epp index 31e45ae4a1..ff24b296e9 100644 --- a/src/dsql/DdlNodes.epp +++ b/src/dsql/DdlNodes.epp @@ -11751,8 +11751,6 @@ void GrantRevokeNode::grantRevoke(thread_db* tdbb, jrd_tra* transaction, const G char priv[2]; priv[1] = '\0'; - AutoSetRestoreFlag trustedDdlFlag(&tdbb->tdbb_flags, TDBB_trusted_ddl, true); - if (isGrant) { AutoCacheRequest request(tdbb, drq_l_grant1, DYN_REQUESTS); diff --git a/src/dsql/Nodes.h b/src/dsql/Nodes.h index 805c159534..edb7c5bc0a 100644 --- a/src/dsql/Nodes.h +++ b/src/dsql/Nodes.h @@ -194,16 +194,14 @@ public: // Set the scratch's transaction when executing a node. Fact of accessing the scratch during // execution is a hack. - void executeDdl(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction) + void executeDdl(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction, bool trusted = false) { // dsqlScratch should be NULL with CREATE DATABASE. if (dsqlScratch) dsqlScratch->setTransaction(transaction); - const ULONG flag = checkPermission(tdbb, transaction) ? TDBB_trusted_ddl : 0; - - Firebird::AutoSetRestoreFlag trustedDdlFlag(&tdbb->tdbb_flags, flag, true); - + if (!trusted) + checkPermission(tdbb, transaction); execute(tdbb, dsqlScratch, transaction); } diff --git a/src/dsql/PackageNodes.epp b/src/dsql/PackageNodes.epp index cec6044573..23f5224451 100644 --- a/src/dsql/PackageNodes.epp +++ b/src/dsql/PackageNodes.epp @@ -404,7 +404,7 @@ bool CreateAlterPackageNode::executeAlter(thread_db* tdbb, DsqlCompilerScratch* DropFunctionNode dropNode(pool, i->name); dropNode.package = name; dropNode.dsqlPass(dsqlScratch); - dropNode.executeDdl(tdbb, dsqlScratch, transaction); + dropNode.executeDdl(tdbb, dsqlScratch, transaction, true); } } @@ -416,7 +416,7 @@ bool CreateAlterPackageNode::executeAlter(thread_db* tdbb, DsqlCompilerScratch* DropProcedureNode dropNode(pool, i->name); dropNode.package = name; dropNode.dsqlPass(dsqlScratch); - dropNode.executeDdl(tdbb, dsqlScratch, transaction); + dropNode.executeDdl(tdbb, dsqlScratch, transaction, true); } } @@ -465,12 +465,12 @@ void CreateAlterPackageNode::executeItems(thread_db* tdbb, DsqlCompilerScratch* { case Item::FUNCTION: (*items)[i].function->packageOwner = owner; - (*items)[i].function->executeDdl(tdbb, (*items)[i].dsqlScratch, transaction); + (*items)[i].function->executeDdl(tdbb, (*items)[i].dsqlScratch, transaction, true); break; case Item::PROCEDURE: (*items)[i].procedure->packageOwner = owner; - (*items)[i].procedure->executeDdl(tdbb, (*items)[i].dsqlScratch, transaction); + (*items)[i].procedure->executeDdl(tdbb, (*items)[i].dsqlScratch, transaction, true); break; } } @@ -549,7 +549,7 @@ void DropPackageNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, DropFunctionNode dropNode(pool, i->name); dropNode.package = name; dropNode.dsqlPass(dsqlScratch); - dropNode.executeDdl(tdbb, dsqlScratch, transaction); + dropNode.executeDdl(tdbb, dsqlScratch, transaction, true); } for (SortedObjectsArray::iterator i = existingProcs.begin(); @@ -558,7 +558,7 @@ void DropPackageNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, DropProcedureNode dropNode(pool, i->name); dropNode.package = name; dropNode.dsqlPass(dsqlScratch); - dropNode.executeDdl(tdbb, dsqlScratch, transaction); + dropNode.executeDdl(tdbb, dsqlScratch, transaction, true); } requestHandle.reset(tdbb, drq_e_pkg_prv, DYN_REQUESTS); @@ -787,7 +787,7 @@ void CreatePackageBodyNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlSc func->packageOwner = owner; func->preserveDefaults = existingFuncs.exist(Signature(func->name)) && arrays[i] == items; - func->executeDdl(tdbb, elem.dsqlScratch, transaction); + func->executeDdl(tdbb, elem.dsqlScratch, transaction, true); break; } @@ -808,7 +808,7 @@ void CreatePackageBodyNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlSc proc->packageOwner = owner; proc->preserveDefaults = existingProcs.exist(Signature(proc->name)) && arrays[i] == items; - proc->executeDdl(tdbb, elem.dsqlScratch, transaction); + proc->executeDdl(tdbb, elem.dsqlScratch, transaction, true); break; } } @@ -935,7 +935,7 @@ void DropPackageBodyNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScra DropFunctionNode dropNode(pool, FUN.RDB$FUNCTION_NAME); dropNode.package = name; dropNode.dsqlPass(dsqlScratch); - dropNode.executeDdl(tdbb, dsqlScratch, transaction); + dropNode.executeDdl(tdbb, dsqlScratch, transaction, true); } else { @@ -962,7 +962,7 @@ void DropPackageBodyNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScra DropProcedureNode dropNode(pool, PRC.RDB$PROCEDURE_NAME); dropNode.package = name; dropNode.dsqlPass(dsqlScratch); - dropNode.executeDdl(tdbb, dsqlScratch, transaction); + dropNode.executeDdl(tdbb, dsqlScratch, transaction, true); } else { diff --git a/src/include/gen/codetext.h b/src/include/gen/codetext.h index fbbfbc78d7..0e65cfb0f4 100644 --- a/src/include/gen/codetext.h +++ b/src/include/gen/codetext.h @@ -965,6 +965,7 @@ static const struct { {"bind_convert", 335545261}, {"cannot_update_old_blob", 335545262}, {"cannot_read_new_blob", 335545263}, + {"dyn_no_create_priv", 335545264}, {"gfix_db_name", 335740929}, {"gfix_invalid_sw", 335740930}, {"gfix_incmp_sw", 335740932}, diff --git a/src/include/gen/iberror.h b/src/include/gen/iberror.h index e61f44c1f7..4e6f933c79 100644 --- a/src/include/gen/iberror.h +++ b/src/include/gen/iberror.h @@ -999,6 +999,7 @@ const ISC_STATUS isc_bind_statement = 335545260L; const ISC_STATUS isc_bind_convert = 335545261L; const ISC_STATUS isc_cannot_update_old_blob = 335545262L; const ISC_STATUS isc_cannot_read_new_blob = 335545263L; +const ISC_STATUS isc_dyn_no_create_priv = 335545264L; const ISC_STATUS isc_gfix_db_name = 335740929L; const ISC_STATUS isc_gfix_invalid_sw = 335740930L; const ISC_STATUS isc_gfix_incmp_sw = 335740932L; @@ -1489,7 +1490,7 @@ const ISC_STATUS isc_trace_switch_user_only = 337182757L; const ISC_STATUS isc_trace_switch_param_miss = 337182758L; const ISC_STATUS isc_trace_param_act_notcompat = 337182759L; const ISC_STATUS isc_trace_mandatory_switch_miss = 337182760L; -const ISC_STATUS isc_err_max = 1433; +const ISC_STATUS isc_err_max = 1434; #else /* c definitions */ @@ -2458,6 +2459,7 @@ const ISC_STATUS isc_err_max = 1433; #define isc_bind_convert 335545261L #define isc_cannot_update_old_blob 335545262L #define isc_cannot_read_new_blob 335545263L +#define isc_dyn_no_create_priv 335545264L #define isc_gfix_db_name 335740929L #define isc_gfix_invalid_sw 335740930L #define isc_gfix_incmp_sw 335740932L @@ -2948,7 +2950,7 @@ const ISC_STATUS isc_err_max = 1433; #define isc_trace_switch_param_miss 337182758L #define isc_trace_param_act_notcompat 337182759L #define isc_trace_mandatory_switch_miss 337182760L -#define isc_err_max 1433 +#define isc_err_max 1434 #endif diff --git a/src/include/gen/msgs.h b/src/include/gen/msgs.h index abb00b140f..86e4745b30 100644 --- a/src/include/gen/msgs.h +++ b/src/include/gen/msgs.h @@ -968,6 +968,7 @@ Data source : @4"}, /* eds_statement */ {335545261, "Can not convert @1 to @2"}, /* bind_convert */ {335545262, "cannot update old BLOB"}, /* cannot_update_old_blob */ {335545263, "cannot read from new BLOB"}, /* cannot_read_new_blob */ + {335545264, "There is no privilege CREATE @1"}, /* dyn_no_create_priv */ {335740929, "data base file name (@1) already given"}, /* gfix_db_name */ {335740930, "invalid switch @1"}, /* gfix_invalid_sw */ {335740932, "incompatible switch combination"}, /* gfix_incmp_sw */ diff --git a/src/include/gen/sql_code.h b/src/include/gen/sql_code.h index b5ac2e1c3d..511f5eb4f8 100644 --- a/src/include/gen/sql_code.h +++ b/src/include/gen/sql_code.h @@ -964,6 +964,7 @@ static const struct { {335545261, -901}, /* 941 bind_convert */ {335545262, -402}, /* 942 cannot_update_old_blob */ {335545263, -402}, /* 943 cannot_read_new_blob */ + {335545264, -901}, /* 944 dyn_no_create_priv */ {335740929, -901}, /* 1 gfix_db_name */ {335740930, -901}, /* 2 gfix_invalid_sw */ {335740932, -901}, /* 4 gfix_incmp_sw */ diff --git a/src/include/gen/sql_state.h b/src/include/gen/sql_state.h index 18bcbdb13a..f7b87a865a 100644 --- a/src/include/gen/sql_state.h +++ b/src/include/gen/sql_state.h @@ -964,6 +964,7 @@ static const struct { {335545261, "22000"}, // 941 bind_convert {335545262, "42000"}, // 942 cannot_update_old_blob {335545263, "42000"}, // 943 cannot_read_new_blob + {335545264, "42000"}, // 944 dyn_no_create_priv {335740929, "00000"}, // 1 gfix_db_name {335740930, "00000"}, // 2 gfix_invalid_sw {335740932, "00000"}, // 4 gfix_incmp_sw diff --git a/src/jrd/jrd.h b/src/jrd/jrd.h index a964ac421d..176caa68c4 100644 --- a/src/jrd/jrd.h +++ b/src/jrd/jrd.h @@ -485,7 +485,6 @@ const ULONG TDBB_use_db_page_space = 128; // use database (not temporary) page const ULONG TDBB_detaching = 256; // detach is in progress const ULONG TDBB_wait_cancel_disable = 512; // don't cancel current waiting operation const ULONG TDBB_cache_unwound = 1024; // page cache was unwound -const ULONG TDBB_trusted_ddl = 2048; // skip DDL permission checks. Set after DDL permission check and clear after DDL execution const ULONG TDBB_reset_stack = 4096; // stack should be reset after stack overflow exception const ULONG TDBB_dfw_cleanup = 8192; // DFW cleanup phase is active const ULONG TDBB_repl_sql = 16384; // SQL statement is being replicated diff --git a/src/jrd/scl.epp b/src/jrd/scl.epp index b1629fe9f0..824ad832ad 100644 --- a/src/jrd/scl.epp +++ b/src/jrd/scl.epp @@ -226,14 +226,6 @@ void SCL_check_access(thread_db* tdbb, **************************************/ SET_TDBB(tdbb); - // RS: SCL_references must be checked for DDL operation of index creation - // and we need to ignore TDBB_trusted_ddl flag in this case. - // More general solution is to remove TDBB_trusted_dll flag since its purpose to - // allow system table modification due DDL operations. It requires removing SCL_checks - // from VIO_{store,erase,motify}. It's quite possible but not so trivial as at first look. - if ((tdbb->tdbb_flags & TDBB_trusted_ddl) && (mask != SCL_references)) - return; - const MetaName& userName = s_class->sclClassUser.second; if (s_class && (s_class->scl_flags & SCL_corrupt)) @@ -295,16 +287,17 @@ void SCL_check_create_access(thread_db* tdbb, int type) Jrd::Attachment* const attachment = tdbb->getAttachment(); // Allow the locksmith any access to database - // TDBB_trusted_ddl flag may be set in CREATE PACKAGE node. I suppose - // if user has CREATE PACKAGE privilege we should not require his to have CREATE FUNCTION as well - if ((tdbb->tdbb_flags & TDBB_trusted_ddl) || - attachment->locksmith(tdbb, SystemPrivilege::MODIFY_ANY_OBJECT_IN_DATABASE)) + if (attachment->locksmith(tdbb, SystemPrivilege::MODIFY_ANY_OBJECT_IN_DATABASE)) return; const SecurityClass::flags_t obj_mask = SCL_get_object_mask(type); if (!(obj_mask & SCL_create)) + { + const char* name = accTypeNumToStr(type); +// ERR_post(Arg::Gds(isc_dyn_no_create_priv) << name); ERR_post(Arg::Gds(isc_dyn_no_priv)); + } } diff --git a/src/jrd/vio.cpp b/src/jrd/vio.cpp index 6c7ff8e06e..ee2ab4b616 100644 --- a/src/jrd/vio.cpp +++ b/src/jrd/vio.cpp @@ -99,7 +99,7 @@ static bool check_nullify_source(thread_db*, record_param*, record_param*, int, static void check_owner(thread_db*, jrd_tra*, record_param*, record_param*, USHORT); static bool check_user(thread_db*, const dsc*); static int check_precommitted(const jrd_tra*, const record_param*); -static void check_rel_field_class(thread_db*, record_param*, SecurityClass::flags_t, jrd_tra*); +static void check_rel_field_class(thread_db*, record_param*, jrd_tra*); static void delete_record(thread_db*, record_param*, ULONG, MemoryPool*); static UCHAR* delete_tail(thread_db*, record_param*, ULONG, UCHAR*, const UCHAR*); static void expunge(thread_db*, record_param*, const jrd_tra*, ULONG); @@ -1557,11 +1557,6 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) case rel_relations: protect_system_table_delupd(tdbb, relation, "DELETE"); - if (EVL_field(0, rpb->rpb_record, f_rel_name, &desc)) - { - SCL_check_relation(tdbb, &desc, SCL_drop); - } - if (EVL_field(0, rpb->rpb_record, f_rel_id, &desc2)) { id = MOV_get_long(tdbb, &desc2, 0); @@ -1569,6 +1564,7 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) { IBERROR(187); // msg 187 cannot delete system relations } + EVL_field(0, rpb->rpb_record, f_rel_name, &desc); DFW_post_work(transaction, dfw_delete_relation, &desc, id); jrd_rel* rel_drop = MET_lookup_relation_id(tdbb, id, false); if (rel_drop) @@ -1578,8 +1574,6 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) case rel_packages: protect_system_table_delupd(tdbb, relation, "DELETE"); - if (EVL_field(0, rpb->rpb_record, f_pkg_name, &desc)) - SCL_check_package(tdbb, &desc, SCL_drop); break; case rel_procedures: @@ -1588,13 +1582,9 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) id = MOV_get_long(tdbb, &desc2, 0); if (EVL_field(0, rpb->rpb_record, f_prc_pkg_name, &desc2)) - { MOV_get_metaname(tdbb, &desc2, package_name); - SCL_check_package(tdbb, &desc2, SCL_drop); - } - if (EVL_field(0, rpb->rpb_record, f_prc_name, &desc) && package_name.isEmpty()) - SCL_check_procedure(tdbb, &desc, SCL_drop); + EVL_field(0, rpb->rpb_record, f_prc_name, &desc); DFW_post_work(transaction, dfw_delete_procedure, &desc, id, package_name); MET_lookup_procedure_id(tdbb, id, false, true, 0); @@ -1602,9 +1592,6 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) case rel_charsets: protect_system_table_delupd(tdbb, relation, "DELETE"); - EVL_field(0, rpb->rpb_record, f_cs_cs_name, &desc); - MOV_get_metaname(tdbb, &desc, object_name); - SCL_check_charset(tdbb, object_name, SCL_drop); break; case rel_collations: @@ -1616,41 +1603,24 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) id = INTL_CS_COLL_TO_TTYPE(id, MOV_get_long(tdbb, &desc2, 0)); EVL_field(0, rpb->rpb_record, f_coll_name, &desc); - MOV_get_metaname(tdbb, &desc, object_name); - SCL_check_collation(tdbb, object_name, SCL_drop); DFW_post_work(transaction, dfw_delete_collation, &desc, id); break; case rel_exceptions: protect_system_table_delupd(tdbb, relation, "DELETE"); EVL_field(0, rpb->rpb_record, f_xcp_name, &desc); - MOV_get_metaname(tdbb, &desc, object_name); - SCL_check_exception(tdbb, object_name, SCL_drop); DFW_post_work(transaction, dfw_delete_exception, &desc, 0); break; case rel_gens: protect_system_table_delupd(tdbb, relation, "DELETE"); EVL_field(0, rpb->rpb_record, f_gen_name, &desc); - MOV_get_metaname(tdbb, &desc, object_name); - SCL_check_generator(tdbb, object_name, SCL_drop); DFW_post_work(transaction, dfw_delete_generator, &desc, 0); break; case rel_funs: protect_system_table_delupd(tdbb, relation, "DELETE"); EVL_field(0, rpb->rpb_record, f_fun_name, &desc); - - if (EVL_field(0, rpb->rpb_record, f_fun_pkg_name, &desc2)) - { - MOV_get_metaname(tdbb, &desc2, package_name); - SCL_check_package(tdbb, &desc2, SCL_drop); - } - else - { - SCL_check_function(tdbb, &desc, SCL_drop); - } - EVL_field(0, rpb->rpb_record, f_fun_id, &desc2); id = MOV_get_long(tdbb, &desc2, 0); @@ -1661,8 +1631,6 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) case rel_indices: protect_system_table_delupd(tdbb, relation, "DELETE"); EVL_field(0, rpb->rpb_record, f_idx_relation, &desc); - EVL_field(0, rpb->rpb_record, f_idx_sys_flag, &desc2); - SCL_check_relation(tdbb, &desc, SCL_control, MOV_get_long(tdbb, &desc2, 0) == 1); EVL_field(0, rpb->rpb_record, f_idx_id, &desc2); if ( (id = MOV_get_long(tdbb, &desc2, 0)) ) { @@ -1720,7 +1688,6 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) case rel_rfr: protect_system_table_delupd(tdbb, relation, "DELETE"); EVL_field(0, rpb->rpb_record, f_rfr_rname, &desc); - SCL_check_relation(tdbb, &desc, SCL_control); DFW_post_work(transaction, dfw_update_format, &desc, 0); EVL_field(0, rpb->rpb_record, f_rfr_fname, &desc2); MOV_get_metaname(tdbb, &desc, object_name); @@ -1734,35 +1701,19 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) case rel_args: protect_system_table_delupd(tdbb, relation, "DELETE"); - if (EVL_field(0, rpb->rpb_record, f_arg_pkg_name, &desc2)) - { - MOV_get_metaname(tdbb, &desc2, package_name); - SCL_check_package(tdbb, &desc2, SCL_control); - } - else - { - EVL_field(0, rpb->rpb_record, f_arg_fun_name, &desc); - SCL_check_function(tdbb, &desc, SCL_control); - } - break; case rel_prc_prms: protect_system_table_delupd(tdbb, relation, "DELETE"); EVL_field(0, rpb->rpb_record, f_prm_procedure, &desc); + MOV_get_metaname(tdbb, &desc, object_name); if (EVL_field(0, rpb->rpb_record, f_prm_pkg_name, &desc2)) { MOV_get_metaname(tdbb, &desc2, package_name); - SCL_check_package(tdbb, &desc2, SCL_control); - } - else - { - SCL_check_procedure(tdbb, &desc, SCL_control); } EVL_field(0, rpb->rpb_record, f_prm_name, &desc2); - MOV_get_metaname(tdbb, &desc, object_name); if ( (procedure = MET_lookup_procedure(tdbb, QualifiedName(object_name, package_name), true)) ) @@ -1780,8 +1731,6 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) case rel_fields: protect_system_table_delupd(tdbb, relation, "DELETE"); EVL_field(0, rpb->rpb_record, f_fld_name, &desc); - MOV_get_metaname(tdbb, &desc, object_name); - SCL_check_domain(tdbb, object_name, SCL_drop); DFW_post_work(transaction, dfw_delete_field, &desc, 0); MET_change_fields(tdbb, transaction, &desc); break; @@ -1789,7 +1738,6 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) case rel_files: protect_system_table_delupd(tdbb, relation, "DELETE"); { - SCL_check_database(tdbb, SCL_alter); const bool name_defined = EVL_field(0, rpb->rpb_record, f_file_name, &desc); const USHORT file_flags = EVL_field(0, rpb->rpb_record, f_file_flags, &desc2) ? MOV_get_long(tdbb, &desc2, 0) : 0; @@ -1822,13 +1770,6 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) case rel_triggers: protect_system_table_delupd(tdbb, relation, "DELETE"); - EVL_field(0, rpb->rpb_record, f_trg_rname, &desc); - - // check if this request go through without checking permissions - if (!(request->getStatement()->flags & JrdStatement::FLAG_IGNORE_PERM)) { - SCL_check_relation(tdbb, &desc, SCL_control); - } - EVL_field(0, rpb->rpb_record, f_trg_rname, &desc2); DFW_post_work(transaction, dfw_update_format, &desc2, 0); EVL_field(0, rpb->rpb_record, f_trg_name, &desc); @@ -2907,10 +2848,11 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j break; case rel_relations: + EVL_field(0, org_rpb->rpb_record, f_rel_name, &desc1); if (!check_nullify_source(tdbb, org_rpb, new_rpb, f_rel_source)) protect_system_table_delupd(tdbb, relation, "UPDATE"); - EVL_field(0, org_rpb->rpb_record, f_rel_name, &desc1); - SCL_check_relation(tdbb, &desc1, SCL_alter); + else + SCL_check_relation(tdbb, &desc1, SCL_alter); check_class(tdbb, transaction, org_rpb, new_rpb, f_rel_class); check_owner(tdbb, transaction, org_rpb, new_rpb, f_rel_owner); DFW_post_work(transaction, dfw_update_format, &desc1, 0); @@ -2919,8 +2861,11 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j case rel_packages: if (!check_nullify_source(tdbb, org_rpb, new_rpb, f_pkg_header_source, f_pkg_body_source)) protect_system_table_delupd(tdbb, relation, "UPDATE"); - if (EVL_field(0, org_rpb->rpb_record, f_pkg_name, &desc1)) - SCL_check_package(tdbb, &desc1, SCL_alter); + else + { + if (EVL_field(0, org_rpb->rpb_record, f_pkg_name, &desc1)) + SCL_check_package(tdbb, &desc1, SCL_alter); + } check_class(tdbb, transaction, org_rpb, new_rpb, f_pkg_class); check_owner(tdbb, transaction, org_rpb, new_rpb, f_pkg_owner); break; @@ -2928,6 +2873,7 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j case rel_procedures: if (!check_nullify_source(tdbb, org_rpb, new_rpb, f_prc_source)) protect_system_table_delupd(tdbb, relation, "UPDATE"); + EVL_field(0, org_rpb->rpb_record, f_prc_name, &desc1); if (EVL_field(0, org_rpb->rpb_record, f_prc_pkg_name, &desc2)) @@ -2936,9 +2882,7 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j SCL_check_package(tdbb, &desc2, SCL_alter); } else - { SCL_check_procedure(tdbb, &desc1, SCL_alter); - } check_class(tdbb, transaction, org_rpb, new_rpb, f_prc_class); check_owner(tdbb, transaction, org_rpb, new_rpb, f_prc_owner); @@ -2952,18 +2896,18 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j break; case rel_funs: + EVL_field(0, org_rpb->rpb_record, f_fun_name, &desc1); if (!check_nullify_source(tdbb, org_rpb, new_rpb, f_fun_source)) protect_system_table_delupd(tdbb, relation, "UPDATE"); - EVL_field(0, org_rpb->rpb_record, f_fun_name, &desc1); - - if (EVL_field(0, org_rpb->rpb_record, f_fun_pkg_name, &desc2)) - { - MOV_get_metaname(tdbb, &desc2, package_name); - SCL_check_package(tdbb, &desc2, SCL_alter); - } else { - SCL_check_function(tdbb, &desc1, SCL_alter); + if (EVL_field(0, org_rpb->rpb_record, f_fun_pkg_name, &desc2)) + { + MOV_get_metaname(tdbb, &desc2, package_name); + SCL_check_package(tdbb, &desc2, SCL_alter); + } + else + SCL_check_function(tdbb, &desc1, SCL_alter); } check_class(tdbb, transaction, org_rpb, new_rpb, f_fun_class); @@ -2979,9 +2923,6 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j case rel_gens: protect_system_table_delupd(tdbb, relation, "UPDATE"); - EVL_field(0, org_rpb->rpb_record, f_gen_name, &desc1); - MOV_get_metaname(tdbb, &desc1, object_name); - SCL_check_generator(tdbb, object_name, SCL_alter); check_class(tdbb, transaction, org_rpb, new_rpb, f_gen_class); check_owner(tdbb, transaction, org_rpb, new_rpb, f_gen_owner); break; @@ -2989,8 +2930,8 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j case rel_rfr: protect_system_table_delupd(tdbb, relation, "UPDATE"); { - check_rel_field_class(tdbb, org_rpb, SCL_control, transaction); - check_rel_field_class(tdbb, new_rpb, SCL_control, transaction); + check_rel_field_class(tdbb, org_rpb, transaction); + check_rel_field_class(tdbb, new_rpb, transaction); check_class(tdbb, transaction, org_rpb, new_rpb, f_rfr_class); bool rc1 = EVL_field(NULL, org_rpb->rpb_record, f_rfr_null_flag, &desc1); @@ -3023,8 +2964,6 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j case rel_fields: protect_system_table_delupd(tdbb, relation, "UPDATE"); EVL_field(0, org_rpb->rpb_record, f_fld_name, &desc1); - MOV_get_metaname(tdbb, &desc1, object_name); - SCL_check_domain(tdbb, object_name, SCL_alter); if (dfw_should_know(tdbb, org_rpb, new_rpb, f_fld_desc, true)) { @@ -3072,7 +3011,6 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j case rel_indices: protect_system_table_delupd(tdbb, relation, "UPDATE"); EVL_field(0, new_rpb->rpb_record, f_idx_relation, &desc1); - SCL_check_relation(tdbb, &desc1, SCL_control, false); if (dfw_should_know(tdbb, org_rpb, new_rpb, f_idx_desc, true)) { @@ -3092,10 +3030,11 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j break; case rel_triggers: + EVL_field(0, new_rpb->rpb_record, f_trg_rname, &desc1); if (!check_nullify_source(tdbb, org_rpb, new_rpb, f_trg_source)) protect_system_table_delupd(tdbb, relation, "UPDATE"); - EVL_field(0, new_rpb->rpb_record, f_trg_rname, &desc1); - SCL_check_relation(tdbb, &desc1, SCL_control); + else + SCL_check_relation(tdbb, &desc1, SCL_control | SCL_alter); if (dfw_should_know(tdbb, org_rpb, new_rpb, f_trg_desc, true)) { @@ -3120,7 +3059,6 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j case rel_files: protect_system_table_delupd(tdbb, relation, "UPDATE"); { - SCL_check_database(tdbb, SCL_alter); SSHORT new_rel_flags, old_rel_flags; EVL_field(0, new_rpb->rpb_record, f_file_name, &desc1); if (EVL_field(0, new_rpb->rpb_record, f_file_flags, &desc2) && @@ -3137,27 +3075,18 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j case rel_charsets: protect_system_table_delupd(tdbb, relation, "UPDATE"); - EVL_field(0, new_rpb->rpb_record, f_cs_cs_name, &desc1); - MOV_get_metaname(tdbb, &desc1, object_name); - SCL_check_charset(tdbb, object_name, SCL_alter); check_class(tdbb, transaction, org_rpb, new_rpb, f_cs_class); check_owner(tdbb, transaction, org_rpb, new_rpb, f_cs_owner); break; case rel_collations: protect_system_table_delupd(tdbb, relation, "UPDATE"); - EVL_field(0, new_rpb->rpb_record, f_coll_name, &desc1); - MOV_get_metaname(tdbb, &desc1, object_name); - SCL_check_collation(tdbb, object_name, SCL_alter); check_class(tdbb, transaction, org_rpb, new_rpb, f_coll_class); check_owner(tdbb, transaction, org_rpb, new_rpb, f_coll_owner); break; case rel_exceptions: protect_system_table_delupd(tdbb, relation, "UPDATE"); - EVL_field(0, new_rpb->rpb_record, f_xcp_name, &desc1); - MOV_get_metaname(tdbb, &desc1, object_name); - SCL_check_exception(tdbb, object_name, SCL_alter); check_class(tdbb, transaction, org_rpb, new_rpb, f_xcp_class); check_owner(tdbb, transaction, org_rpb, new_rpb, f_xcp_owner); break; @@ -3629,9 +3558,6 @@ void VIO_store(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) case rel_indices: protect_system_table_insert(tdbb, request, relation); - EVL_field(0, rpb->rpb_record, f_idx_relation, &desc); - EVL_field(0, rpb->rpb_record, f_idx_sys_flag, &desc2); - SCL_check_relation(tdbb, &desc, SCL_control, MOV_get_long(tdbb, &desc2, 0) == 1); EVL_field(0, rpb->rpb_record, f_idx_name, &desc); if (EVL_field(0, rpb->rpb_record, f_idx_exp_blr, &desc2)) { @@ -3647,7 +3573,6 @@ void VIO_store(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) case rel_rfr: protect_system_table_insert(tdbb, request, relation); EVL_field(0, rpb->rpb_record, f_rfr_rname, &desc); - SCL_check_relation(tdbb, &desc, SCL_control); DFW_post_work(transaction, dfw_update_format, &desc, 0); set_system_flag(tdbb, rpb->rpb_record, f_rfr_sys_flag); break; @@ -3714,7 +3639,7 @@ void VIO_store(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) // check if this request go through without checking permissions if (!(request->getStatement()->flags & JrdStatement::FLAG_IGNORE_PERM)) - SCL_check_relation(tdbb, &desc, SCL_control); + SCL_check_relation(tdbb, &desc, SCL_control | SCL_alter); if (EVL_field(0, rpb->rpb_record, f_trg_rname, &desc2)) DFW_post_work(transaction, dfw_update_format, &desc2, 0); @@ -4158,7 +4083,6 @@ static int check_precommitted(const jrd_tra* transaction, const record_param* rp static void check_rel_field_class(thread_db* tdbb, record_param* rpb, - SecurityClass::flags_t flags, jrd_tra* transaction) { /********************************************* @@ -4175,34 +4099,8 @@ static void check_rel_field_class(thread_db* tdbb, **************************************/ SET_TDBB(tdbb); - bool okField = true; DSC desc; - if (EVL_field(0, rpb->rpb_record, f_rfr_class, &desc)) - { - const Firebird::MetaName class_name(reinterpret_cast(desc.dsc_address), - desc.dsc_length); - const SecurityClass* s_class = SCL_get_class(tdbb, class_name.c_str()); - if (s_class) - { - // In case when user has no access to the field, - // he may have access to relation as whole. - try - { - SCL_check_access(tdbb, s_class, 0, NULL, flags, SCL_object_column, false, ""); - } - catch (const Firebird::Exception&) - { - fb_utils::init_status(tdbb->tdbb_status_vector); - okField = false; - } - } - } - EVL_field(0, rpb->rpb_record, f_rfr_rname, &desc); - - if (!okField) - SCL_check_relation(tdbb, &desc, flags); - DFW_post_work(transaction, dfw_update_format, &desc, 0); } diff --git a/src/msgs/facilities2.sql b/src/msgs/facilities2.sql index 784e64b79a..d5fee06757 100644 --- a/src/msgs/facilities2.sql +++ b/src/msgs/facilities2.sql @@ -1,7 +1,7 @@ /* MAX_NUMBER is the next number to be used, always one more than the highest message number. */ set bulk_insert INSERT INTO FACILITIES (LAST_CHANGE, FACILITY, FAC_CODE, MAX_NUMBER) VALUES (?, ?, ?, ?); -- -('2019-12-19 12:10:00', 'JRD', 0, 944) +('2019-12-19 12:10:00', 'JRD', 0, 945) ('2015-03-17 18:33:00', 'QLI', 1, 533) ('2018-03-17 12:00:00', 'GFIX', 3, 136) ('1996-11-07 13:39:40', 'GPRE', 4, 1) diff --git a/src/msgs/messages2.sql b/src/msgs/messages2.sql index 0aee80450b..680008c239 100644 --- a/src/msgs/messages2.sql +++ b/src/msgs/messages2.sql @@ -1051,6 +1051,7 @@ Data source : @4', NULL, NULL) ('bind_convert', NULL, 'Coercion.cpp', NULL, 0, 941, NULL, 'Can not convert @1 to @2', NULL, NULL); ('cannot_update_old_blob', 'BLB_put_segment', 'blb.cpp', NULL, 0, 942, NULL, 'cannot update old BLOB', NULL, NULL); ('cannot_read_new_blob', 'BLB_get_segment', 'blb.cpp', NULL, 0, 943, NULL, 'cannot read from new BLOB', NULL, NULL); +('dyn_no_create_priv', NULL, 'scl.epp', NULL, 0, 944, NULL, 'There is no privilege CREATE @1', NULL, NULL); -- QLI (NULL, NULL, NULL, NULL, 1, 0, NULL, 'expected type', NULL, NULL); (NULL, NULL, NULL, NULL, 1, 1, NULL, 'bad block type', NULL, NULL); diff --git a/src/msgs/system_errors2.sql b/src/msgs/system_errors2.sql index 9103851695..f904a2ec09 100644 --- a/src/msgs/system_errors2.sql +++ b/src/msgs/system_errors2.sql @@ -950,6 +950,7 @@ set bulk_insert INSERT INTO SYSTEM_ERRORS (SQL_CODE, SQL_CLASS, SQL_SUBCLASS, FA (-901, '22', '000', 0, 941, 'bind_convert', NULL, NULL) (-402, '42', '000', 0, 942, 'cannot_update_old_blob', NULL, NULL) (-402, '42', '000', 0, 943, 'cannot_read_new_blob', NULL, NULL) +(-901, '42', '000', 0, 944, 'dyn_no_create_priv', NULL, NULL) -- GFIX (-901, '00', '000', 3, 1, 'gfix_db_name', NULL, NULL) (-901, '00', '000', 3, 2, 'gfix_invalid_sw', NULL, NULL) From 675e7e0171ad317409e4cefc1c4fe23e741b45dd Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 11 Jan 2020 00:04:20 +0000 Subject: [PATCH 205/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 532c36683a..12fc8a0e02 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1715 + FORMAL BUILD NUMBER:1717 */ -#define PRODUCT_VER_STRING "4.0.0.1715" -#define FILE_VER_STRING "WI-T4.0.0.1715" -#define LICENSE_VER_STRING "WI-T4.0.0.1715" -#define FILE_VER_NUMBER 4, 0, 0, 1715 +#define PRODUCT_VER_STRING "4.0.0.1717" +#define FILE_VER_STRING "WI-T4.0.0.1717" +#define LICENSE_VER_STRING "WI-T4.0.0.1717" +#define FILE_VER_NUMBER 4, 0, 0, 1717 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1715" +#define FB_BUILD_NO "1717" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 1f6cbcf0ce..0e0c7eb584 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1715 +BuildNum=1717 NowAt=`pwd` cd `dirname $0` From a9923c92afbfed3d3b41423f5f29b754e00bcc09 Mon Sep 17 00:00:00 2001 From: Dmitry Yemanov Date: Sun, 12 Jan 2020 10:38:31 +0300 Subject: [PATCH 206/274] Avoid unnecessary operations in the destructor. This also prevents possible hangs in Classic builds. --- src/jrd/Monitoring.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/jrd/Monitoring.cpp b/src/jrd/Monitoring.cpp index f5d55d1bbe..7a9986cf9a 100644 --- a/src/jrd/Monitoring.cpp +++ b/src/jrd/Monitoring.cpp @@ -133,13 +133,20 @@ MonitoringData::MonitoringData(const Database* dbb) MonitoringData::~MonitoringData() { - Guard guard(this); + m_sharedMemory->mutexLock(); - if (m_sharedMemory->getHeader() && - m_sharedMemory->getHeader()->used == alignOffset(sizeof(Header))) + try { - m_sharedMemory->removeMapFile(); + if (m_sharedMemory->getHeader() && + m_sharedMemory->getHeader()->used == alignOffset(sizeof(Header))) + { + m_sharedMemory->removeMapFile(); + } } + catch (const Exception&) + {} // no-op + + m_sharedMemory->mutexUnlock(); } From fb74077793d1e8acd171035b8f99892c1fc25c29 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sun, 12 Jan 2020 13:29:06 -0300 Subject: [PATCH 207/274] Avoid warning of possible usage without initialization. --- src/common/cvt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/cvt.cpp b/src/common/cvt.cpp index ecac568f9f..dfeb2019e2 100644 --- a/src/common/cvt.cpp +++ b/src/common/cvt.cpp @@ -1802,7 +1802,7 @@ void CVT_move_common(const dsc* from, dsc* to, DecimalStatus decSt, Callbacks* c const UCHAR* start = to->dsc_address; UCHAR fill_char = ASCII_SPACE; Jrd::CharSet* toCharset = cb->getToCharset(charset2); - ULONG toLength; + ULONG toLength = 0; ULONG fill; if (charset2 == ttype_binary) From d0bebcb7893b9ef9cbaef430c89db18521e3b8d7 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Mon, 13 Jan 2020 00:04:51 +0000 Subject: [PATCH 208/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 12fc8a0e02..f160c1f5e4 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1717 + FORMAL BUILD NUMBER:1719 */ -#define PRODUCT_VER_STRING "4.0.0.1717" -#define FILE_VER_STRING "WI-T4.0.0.1717" -#define LICENSE_VER_STRING "WI-T4.0.0.1717" -#define FILE_VER_NUMBER 4, 0, 0, 1717 +#define PRODUCT_VER_STRING "4.0.0.1719" +#define FILE_VER_STRING "WI-T4.0.0.1719" +#define LICENSE_VER_STRING "WI-T4.0.0.1719" +#define FILE_VER_NUMBER 4, 0, 0, 1719 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1717" +#define FB_BUILD_NO "1719" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 0e0c7eb584..d8a847a2d2 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1717 +BuildNum=1719 NowAt=`pwd` cd `dirname $0` From 190262f63410022f261aef380e6c1393a93c3582 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Mon, 13 Jan 2020 17:07:26 +0300 Subject: [PATCH 209/274] Disable simultaneous access to read-only database from multiple processes in SS mode --- src/jrd/os/posix/unix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jrd/os/posix/unix.cpp b/src/jrd/os/posix/unix.cpp index b1c5ab06fb..3a0794dbae 100644 --- a/src/jrd/os/posix/unix.cpp +++ b/src/jrd/os/posix/unix.cpp @@ -713,7 +713,7 @@ jrd_file* PIO_open(thread_db* tdbb, } const bool shareMode = dbb->dbb_config->getServerMode() != MODE_SUPER; - lockDatabaseFile(desc, shareMode || readOnly, false, file_name.c_str(), isc_io_open_err); + lockDatabaseFile(desc, shareMode, false, file_name.c_str(), isc_io_open_err); // os_utils::posix_fadvise(desc, 0, 0, POSIX_FADV_RANDOM); From fa8d7c984a1fa7f6d32c592ab462775145d86a9e Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Mon, 13 Jan 2020 17:08:23 +0300 Subject: [PATCH 210/274] Make firebird engine use classic mode during boot build --- src/common/config/config.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/config/config.cpp b/src/common/config/config.cpp index c02df718e1..ddfe804adb 100644 --- a/src/common/config/config.cpp +++ b/src/common/config/config.cpp @@ -202,7 +202,7 @@ const Config::ConfigEntry Config::entries[MAX_CONFIG_KEY] = {TYPE_STRING, "UserManager", (ConfigValue) "Srp"}, {TYPE_STRING, "TracePlugin", (ConfigValue) "fbtrace"}, {TYPE_STRING, "SecurityDatabase", (ConfigValue) "security.db"}, // sec/db alias - rely on databases.conf - {TYPE_STRING, "ServerMode", (ConfigValue) "Super"}, + {TYPE_STRING, "ServerMode", (ConfigValue) ""}, // actual value differs in boot/regular cases {TYPE_STRING, "WireCrypt", (ConfigValue) NULL}, {TYPE_STRING, "WireCryptPlugin", (ConfigValue) "Arc4"}, {TYPE_STRING, "KeyHolderPlugin", (ConfigValue) ""}, @@ -718,7 +718,7 @@ int Config::getServerMode() } // use default - rc = MODE_SUPER; + rc = fb_utils::bootBuild() ? MODE_CLASSIC : MODE_SUPER; return rc; } From 9d02e7e1e4d1e411edc343fa7185394f08ad964c Mon Sep 17 00:00:00 2001 From: Kovalenko Dmitry Date: Mon, 13 Jan 2020 10:20:06 +0300 Subject: [PATCH 211/274] Correction of RefPtr::assign When RefPtr::assign returns to caller it (object) may be already destroyed. As result "return ptr" will access (read 'ptr' member) the destroyed object. The correct code - 'return p;' Note: usually smart pointers do not return internal/raw pointers. They return reference to himself. --- src/common/classes/RefCounted.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/classes/RefCounted.h b/src/common/classes/RefCounted.h index d8a6027c47..547f1f9171 100644 --- a/src/common/classes/RefCounted.h +++ b/src/common/classes/RefCounted.h @@ -219,7 +219,7 @@ namespace Firebird } } - return ptr; + return p; } private: From 5e962f0e5a363f5ef1e88c22c41d2588abf2d33b Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Mon, 13 Jan 2020 20:22:51 +0300 Subject: [PATCH 212/274] Fixed CORE-2251: gbak doesn't return error code --- src/burp/burp.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/burp/burp.cpp b/src/burp/burp.cpp index c8bc046f6e..ed449de7f4 100644 --- a/src/burp/burp.cpp +++ b/src/burp/burp.cpp @@ -826,16 +826,14 @@ int gbak(Firebird::UtilSvc* uSvc) FILE* tmp_outfile = os_utils::fopen(redirect, fopen_read_type); if (tmp_outfile) { - BURP_print(true, 66, redirect); - // msg 66 can't open status and error output file %s fclose(tmp_outfile); - BURP_exit_local(FINI_ERROR, tdgbl); + BURP_error(66, true, SafeArg() << redirect); + // msg 66 can't open status and error output file %s } if (! (tdgbl->output_file = os_utils::fopen(redirect, fopen_write_type))) { - BURP_print(true, 66, redirect); + BURP_error(66, true, SafeArg() << redirect); // msg 66 can't open status and error output file %s - BURP_exit_local(FINI_ERROR, tdgbl); } } } From 96c59e65408395561a2dda947afdac39d6c16cd0 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Tue, 14 Jan 2020 00:04:36 +0000 Subject: [PATCH 213/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index f160c1f5e4..f9937727b1 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1719 + FORMAL BUILD NUMBER:1723 */ -#define PRODUCT_VER_STRING "4.0.0.1719" -#define FILE_VER_STRING "WI-T4.0.0.1719" -#define LICENSE_VER_STRING "WI-T4.0.0.1719" -#define FILE_VER_NUMBER 4, 0, 0, 1719 +#define PRODUCT_VER_STRING "4.0.0.1723" +#define FILE_VER_STRING "WI-T4.0.0.1723" +#define LICENSE_VER_STRING "WI-T4.0.0.1723" +#define FILE_VER_NUMBER 4, 0, 0, 1723 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1719" +#define FB_BUILD_NO "1723" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index d8a847a2d2..73bb6e26c3 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1719 +BuildNum=1723 NowAt=`pwd` cd `dirname $0` From c6523020b15eebdcb9eef47d04ade8594eaa8439 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Tue, 14 Jan 2020 17:10:48 +0300 Subject: [PATCH 214/274] Fixed CORE-6227: isc_info_svc_user_dbpath always returns alias of main security database --- src/jrd/svc.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/jrd/svc.cpp b/src/jrd/svc.cpp index f31d051e0e..e1861ad296 100644 --- a/src/jrd/svc.cpp +++ b/src/jrd/svc.cpp @@ -1365,11 +1365,13 @@ ISC_STATUS Service::query2(thread_db* /*tdbb*/, case isc_info_svc_user_dbpath: if (svc_user_flag & SVC_user_dba) { - // The path to the user security database (security2.fdb) - const RefPtr defConf(Config::getDefaultConfig()); - const char* secDb = defConf->getSecurityDatabase(); + // The path to the user security database + PathName secDb; + RefPtr config; + expandDatabaseName(svc_expected_db, secDb, &config); + expandDatabaseName(config->getSecurityDatabase(), secDb, nullptr); - if (!(info = INF_put_item(item, static_cast(strlen(secDb)), secDb, info, end))) + if (!(info = INF_put_item(item, static_cast(secDb.length()), secDb.c_str(), info, end))) { return 0; } @@ -1817,11 +1819,13 @@ void Service::query(USHORT send_item_length, case isc_info_svc_user_dbpath: if (svc_user_flag & SVC_user_dba) { - // The path to the user security database (security2.fdb) - const RefPtr defConf(Config::getDefaultConfig()); - const char* secDb = defConf->getSecurityDatabase(); + // The path to the user security database + PathName secDb; + RefPtr config; + expandDatabaseName(svc_expected_db, secDb, &config); + expandDatabaseName(config->getSecurityDatabase(), secDb, nullptr); - if (!(info = INF_put_item(item, static_cast(strlen(secDb)), secDb, info, end))) + if (!(info = INF_put_item(item, static_cast(secDb.length()), secDb.c_str(), info, end))) { return; } From 64ecbd8dd2026baf6954c3ce09e77bbf85613a9e Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Tue, 14 Jan 2020 12:40:25 -0300 Subject: [PATCH 215/274] CORE-6214 - Update outdated tzdata version. Added documentation and script for update. zipjs.bat is downloaded from https://github.com/npocmaka/batch.scripts/blob/master/hybrids/jscript/zipjs.bat --- builds/posix/Makefile.in | 10 +- builds/win32/make_all.bat | 2 + builds/win32/make_icu.bat | 8 +- builds/win32/zipjs.bat | 823 +++++++++++++++++++++++++ doc/sql.extensions/README.time_zone.md | 12 +- src/common/TimeZoneUtil.cpp | 16 + src/common/TimeZoneUtil.h | 6 + src/common/unicode_util.cpp | 13 + src/common/utils.cpp | 30 + src/common/utils_proto.h | 1 + src/yvalve/why.cpp | 14 + tzdata/be.zip | Bin 0 -> 91876 bytes tzdata/le.zip | Bin 0 -> 91147 bytes tzdata/update.sh | 32 + tzdata/version.txt | 1 + 15 files changed, 959 insertions(+), 9 deletions(-) create mode 100644 builds/win32/zipjs.bat create mode 100644 tzdata/be.zip create mode 100644 tzdata/le.zip create mode 100755 tzdata/update.sh create mode 100644 tzdata/version.txt diff --git a/builds/posix/Makefile.in b/builds/posix/Makefile.in index 538c3fb714..c16801ea95 100644 --- a/builds/posix/Makefile.in +++ b/builds/posix/Makefile.in @@ -630,13 +630,13 @@ $(IDS): $(SRC_ROOT)/misc/ids.m $(SRC_ROOT)/jrd/relations.h # all the rest we need to build # -.PHONY: qli message_file gbak_files +.PHONY: qli message_file tzdata gbak_files FDB_FILES := $(HELP_FDB) $(ROOT)/gen/msg.fdb $(SECURITY_FDB) $(FIREBIRD)/examples/empbuild/employee.fdb GBAK_FILES := $(FDB_FILES:.fdb=.gbak) $(FIREBIRD)/msg.gbak GBAK_FILES := $(subst Native,$(TARGET),$(GBAK_FILES)) -rest: qli message_file +rest: qli message_file tzdata cross_rest: qli gbak_files $(MAKE) $(BUILD_FILE) @@ -656,6 +656,12 @@ $(FIREBIRD_MSG): $(BUILD_FILE) msg.timestamp $(BUILD_FILE) -d msg.fdb -f $@ $(CHMOD_6) $@ +tzdata: $(FIREBIRD)/tzdata + +# FIXME: For big-endian, be.zip must be used. +$(FIREBIRD)/tzdata: $(ROOT)/tzdata/le.zip + unzip -o $(ROOT)/tzdata/le.zip -d $(FIREBIRD)/tzdata + $(BUILD_FILE): $(BUILD_Objects) $(COMMON_LIB) $(EXE_LINK) $(EXE_LINK_OPTIONS) $(LSB_UNDEF) $^ -o $@ $(FIREBIRD_LIBRARY_LINK) $(LINK_LIBS) $(call LINK_DARWIN_RPATH,..) diff --git a/builds/win32/make_all.bat b/builds/win32/make_all.bat index b10caa714c..d1f6ec1b7f 100644 --- a/builds/win32/make_all.bat +++ b/builds/win32/make_all.bat @@ -38,6 +38,7 @@ if errorlevel 1 call :ERROR build failed - see make_all_%FB_TARGET_PLATFORM%.log @mkdir %FB_OUTPUT_DIR% 2>nul @mkdir %FB_OUTPUT_DIR%\intl 2>nul +@mkdir %FB_OUTPUT_DIR%\tzdata 2>nul @mkdir %FB_OUTPUT_DIR%\help 2>nul @mkdir %FB_OUTPUT_DIR%\doc 2>nul @mkdir %FB_OUTPUT_DIR%\doc\sql.extensions 2>nul @@ -50,6 +51,7 @@ if errorlevel 1 call :ERROR build failed - see make_all_%FB_TARGET_PLATFORM%.log @copy %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\* %FB_OUTPUT_DIR% >nul @copy %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\intl\* %FB_OUTPUT_DIR%\intl >nul +@copy %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\tzdata\* %FB_OUTPUT_DIR%\tzdata >nul @copy %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\system32\* %FB_OUTPUT_DIR%\system32 >nul @copy %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\plugins\*.dll %FB_OUTPUT_DIR%\plugins >nul @copy %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\plugins\udr\*.dll %FB_OUTPUT_DIR%\plugins\udr >nul diff --git a/builds/win32/make_icu.bat b/builds/win32/make_icu.bat index 3f14be2659..7eaa3aaf0a 100644 --- a/builds/win32/make_icu.bat +++ b/builds/win32/make_icu.bat @@ -10,12 +10,14 @@ :: MAIN @echo Extracting pre-built ICU - %FB_ROOT_PATH%\extern\icu\icu.exe -y > make_icu_%FB_TARGET_PLATFORM%.log 2>&1 - if errorlevel 1 call :ERROR build failed - see make_icu_%FB_TARGET_PLATFORM%.log for details -@goto :EOF +@echo Extracting tzdata +mkdir %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\tzdata +call zipjs.bat unzip -source "%FB_LONG_ROOT_PATH%\tzdata\le.zip" -destination %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\tzdata -keep yes + +@goto :EOF :ERROR diff --git a/builds/win32/zipjs.bat b/builds/win32/zipjs.bat new file mode 100644 index 0000000000..02b8fcbc24 --- /dev/null +++ b/builds/win32/zipjs.bat @@ -0,0 +1,823 @@ +@if (@X)==(@Y) @end /* JScript comment + @echo off + + rem :: the first argument is the script name as it will be used for proper help message + cscript //E:JScript //nologo "%~f0" "%~nx0" %* + + exit /b %errorlevel% + +@if (@X)==(@Y) @end JScript comment */ + + +/* +Compression/uncompression command-line tool that uses Shell.Application and WSH/Jscript - +http://msdn.microsoft.com/en-us/library/windows/desktop/bb774085(v=vs.85).aspx + +Some resources That I've used: +http://www.robvanderwoude.com/vbstech_files_zip.php +https://code.google.com/p/jsxt/source/browse/trunk/js/win32/ZipFile.js?r=161 + + + + +UPDATE *17-03-15* + +Devnullius Plussed noticed a bug in ZipDirItems and ZipItem functions (now fixed) +And also following issues (at the moment not handled by the script): +- if there's not enough space on the system drive (usually C:\) the script could produce various errors , most often the script halts. +- Folders and files that contain unicode symbols cannot be handled by Shell.Application object. + +UPDATE *24-03-15* + +Error messages are caught in waitforcount method and if shuch pops-up the script is stopped. +As I don't know hoe to check the content of the pop-up the exact reason for the failure is not given +but only the possible reasons. + +UPDATE *22-02-16* + +Javid Pack(https://github.com/JavidPack) has found two bugs in zipItem command and in ZipItem function.Now fixed. + +------ +It's possible to be ported for C#,Powershell and JScript.net so I'm planning to do it at some time. + +For sure there's a lot of room for improvements and optimization and I'm absolutely sure there are some bugs +as the script is big enough to not have. + + + +!!! +For suggestions contact me at - npocmaka@gmail.com +!!! + +*/ + + +////////////////////////////////////// +// CONSTANTS + +// TODO - Shell.Application and Scripting.FileSystemObject objects could be set as global variables to avoid theit creation +// in every method. + +//empty zip character sequense +var ZIP_DATA= "PK" + String.fromCharCode(5) + String.fromCharCode(6) + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; + +var SLEEP_INTERVAL=200; + +//copy option(s) used by Shell.Application.CopyHere/MoveHere +var NO_PROGRESS_BAR=4; + + +//oprions used for zip/unzip +var force=true; +var move=false; + +//option used for listing content of archive +var flat=false; + +var source=""; +var destination=""; + +var ARGS = WScript.Arguments; +var scriptName=ARGS.Item(0); + +// +////////////////////////////////////// + +////////////////////////////////////// +// ADODB.Stream extensions + +if ( ! this.ADODB ) { + var ADODB = {}; +} + +if ( ! ADODB.Stream ) { + ADODB.Stream = {}; +} + +// writes a binary data to a file +if ( ! ADODB.Stream.writeFile ) { + ADODB.Stream.writeFile = function(filename, bindata) + { + var stream = new ActiveXObject("ADODB.Stream"); + stream.Type = 2; + stream.Mode = 3; + stream.Charset ="ASCII"; + stream.Open(); + stream.Position = 0; + stream.WriteText(bindata); + stream.SaveToFile(filename, 2); + stream.Close(); + return true; + }; +} + +// +////////////////////////////////////// + +////////////////////////////////////// +// common + +if ( ! this.Common ) { + var Common = {}; +} + +if ( ! Common.WaitForCount ) { + Common.WaitForCount = function(folderObject,targetCount,countFunction){ + var shell = new ActiveXObject("Wscript.Shell"); + while (countFunction(folderObject) < targetCount ){ + WScript.Sleep(SLEEP_INTERVAL); + //checks if a pop-up with error message appears while zipping + //at the moment I have no idea how to read the pop-up content + // to give the exact reason for failing + if (shell.AppActivate("Compressed (zipped) Folders Error")) { + WScript.Echo("Error While zipping"); + WScript.Echo(""); + WScript.Echo("Possible reasons:"); + WScript.Echo(" -source contains filename(s) with unicode characters"); + WScript.Echo(" -produces zip exceeds 8gb size (or 2,5 gb for XP and 2003)"); + WScript.Echo(" -not enough space on system drive (usually C:\\)"); + WScript.Quit(432); + } + + } + } +} + +if ( ! Common.getParent ) { + Common.getParent = function(path){ + var splitted=path.split("\\"); + var result=""; + for (var s=0;s/tzdata` is the default directory where Firebird looks for the database. It could be overriden with the `ICU_TIMEZONE_FILES_DIR` environment variable. + +Important note: Firebird stores `WITH TIME ZONE` values translated to UTC time. If a value is created with one time zone database and later that database is updated and the update changes the information in the range of a stored value, when reading that value it will be returned as a different value than the one initially stored. # Appendix: time zone regions diff --git a/src/common/TimeZoneUtil.cpp b/src/common/TimeZoneUtil.cpp index 05d1d6f7ee..862686466c 100644 --- a/src/common/TimeZoneUtil.cpp +++ b/src/common/TimeZoneUtil.cpp @@ -34,6 +34,7 @@ #include "../common/classes/timestamp.h" #include "../common/classes/GenericMap.h" #include "../common/config/config.h" +#include "../common/os/path_utils.h" #include "unicode/ucal.h" #ifdef TZ_UPDATE @@ -190,6 +191,21 @@ static InitInstance timeZoneStartup; //------------------------------------- const char TimeZoneUtil::GMT_FALLBACK[5] = "GMT*"; +InitInstance TimeZoneUtil::tzDataPath; + +void TimeZoneUtil::initTimeZoneEnv() +{ + PathName path; + PathUtils::concatPath(path, Config::getRootDirectory(), "tzdata"); + + if (fb_utils::setenv("ICU_TIMEZONE_FILES_DIR", path.c_str(), false)) + tzDataPath() = path; +} + +const PathName& TimeZoneUtil::getTzDataPath() +{ + return tzDataPath(); +} // Return the current user's time zone. USHORT TimeZoneUtil::getSystemTimeZone() diff --git a/src/common/TimeZoneUtil.h b/src/common/TimeZoneUtil.h index 69bc5ba18a..b2bda3dd63 100644 --- a/src/common/TimeZoneUtil.h +++ b/src/common/TimeZoneUtil.h @@ -65,6 +65,9 @@ public: static const unsigned MAX_LEN = 32; static const unsigned MAX_SIZE = MAX_LEN + 1; +private: + static InitInstance tzDataPath; + public: static UDate ticksToIcuDate(SINT64 ticks) { @@ -76,6 +79,9 @@ public: return (SINT64(icuDate) * 10) + (TimeStamp::UNIX_DATE * TimeStamp::ISC_TICKS_PER_DAY); } + static void initTimeZoneEnv(); + static const PathName& getTzDataPath(); + static USHORT getSystemTimeZone(); static void getDatabaseVersion(Firebird::string& str); diff --git a/src/common/unicode_util.cpp b/src/common/unicode_util.cpp index faacf19c78..d53dcb50ad 100644 --- a/src/common/unicode_util.cpp +++ b/src/common/unicode_util.cpp @@ -31,12 +31,14 @@ #include "../common/isc_proto.h" #include "../common/CharSet.h" #include "../common/IntlUtil.h" +#include "../common/TimeZoneUtil.h" #include "../common/gdsassert.h" #include "../common/classes/auto.h" #include "../common/classes/GenericMap.h" #include "../common/classes/init.h" #include "../common/classes/objects_array.h" #include "../common/classes/rwlock.h" +#include "../common/config/config.h" #include "../common/StatusHolder.h" #include "../common/os/path_utils.h" @@ -126,9 +128,11 @@ public: void BaseICU::initialize(ModuleLoader::Module* module) { void (U_EXPORT2 *uInit)(UErrorCode* status); + void (U_EXPORT2 *uSetTimeZoneFilesDirectory)(const char* path, UErrorCode* status); void (U_EXPORT2 *uSetDataDirectory)(const char* directory); getEntryPoint("u_init", module, uInit, true); + getEntryPoint("u_setTimeZoneFilesDirectory", module, uSetTimeZoneFilesDirectory, true); getEntryPoint("u_setDataDirectory", module, uSetDataDirectory, true); #if defined(WIN_NT) || defined(DARWIN) @@ -166,6 +170,15 @@ void BaseICU::initialize(ModuleLoader::Module* module) (Arg::Gds(isc_random) << diag).raise(); } } + + // ICU's u_setTimeZoneFilesDirectory is an internal API, but we try to use + // it because internally set ICU_TIMEZONE_FILES_DIR envvar in Windows is not + // safe. See comments in fb_utils::setenv. + if (uSetTimeZoneFilesDirectory && TimeZoneUtil::getTzDataPath().hasData()) + { + UErrorCode status = U_ZERO_ERROR; + uSetTimeZoneFilesDirectory(TimeZoneUtil::getTzDataPath().c_str(), &status); + } } } diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 7ae7234641..09285a6e10 100644 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -38,6 +38,7 @@ #endif #include #include +#include #include #include "../common/gdsassert.h" @@ -303,6 +304,35 @@ bool readenv(const char* env_name, Firebird::PathName& env_value) } +bool setenv(const char* name, const char* value, bool overwrite) +{ +#ifdef WIN_NT + int errcode = 0; + + if (!overwrite) + { + size_t envsize = 0; + errcode = getenv_s(&envsize, NULL, 0, name); + if (errcode || envsize) + return false; + } + + // In Windows, _putenv_s sets only the environment data in the CRT. + // Each DLL (for example ICU) may use a different CRT which different data + // or use Win32's GetEnvironmentVariable, so we also use SetEnvironmentVariable. + // This is a mess and is not guarenteed to work correctly in all situations. + if (SetEnvironmentVariable(name, value)) + { + _putenv_s(name, value); + return true; + } + else + return false; +#else + return ::setenv(name, value, (int) overwrite) == 0; +#endif +} + // *************** // s n p r i n t f // *************** diff --git a/src/common/utils_proto.h b/src/common/utils_proto.h index fa362bec68..f49b7ff1bf 100644 --- a/src/common/utils_proto.h +++ b/src/common/utils_proto.h @@ -55,6 +55,7 @@ namespace fb_utils int name_length_limit(const TEXT* const name, size_t bufsize); bool readenv(const char* env_name, Firebird::string& env_value); bool readenv(const char* env_name, Firebird::PathName& env_value); + bool setenv(const char* name, const char* value, bool overwrite); int snprintf(char* buffer, size_t count, const char* format...); char* cleanup_passwd(char* arg); inline char* get_passwd(char* arg) diff --git a/src/yvalve/why.cpp b/src/yvalve/why.cpp index 1df43ca04b..1fc9754034 100644 --- a/src/yvalve/why.cpp +++ b/src/yvalve/why.cpp @@ -40,6 +40,7 @@ #include "../common/StatementMetadata.h" #include "../common/StatusHolder.h" #include "../common/ThreadStart.h" +#include "../common/TimeZoneUtil.h" #include "../common/isc_proto.h" #include "../common/isc_f_proto.h" #include "../common/utils_proto.h" @@ -737,6 +738,19 @@ RefPtr translateHandle(GlobalPtr //------------------------------------- +class TimeZoneDataInit +{ +public: + explicit TimeZoneDataInit(MemoryPool&) + { + TimeZoneUtil::initTimeZoneEnv(); + } +}; + +static GlobalPtr timeZoneDataInit; + +//------------------------------------- + const int SHUTDOWN_TIMEOUT = 5000; // 5 sec class ShutdownInit diff --git a/tzdata/be.zip b/tzdata/be.zip new file mode 100644 index 0000000000000000000000000000000000000000..410ebd0605b21684d7dbc5fdb5e33bc3dc19f4a1 GIT binary patch literal 91876 zcmaf)Wl$YK+oo~X5Zon5aCZrs-~@MfcZUGM-QC^Y;o$D>?gzJnb2i`mZEfvV?bcR3 zeNE3ZcX#!Vsj2Siz7=JlVZK5@Kp;TCSVXUa33Cb!0r4NWXiYht^EnO67Jv8rBhY)nmZ{CId02WS zpJ2PT9P7%k4gg8QD%Bw*Qn-lp16*3xPJZ2qdS>`o?8hctWf35Ma3XMY5Rwucn)<)= z_#1*JS94c$*9t9${mm@E(8%C7RMsFy>ESMJQt93 z8!NrtK1OTUxwPwslB!8HK*KpD7o7Kk_HBNczUHA-$(G3UGb=V9$8)o10nyiMhaqw$ zHVp;~n3!0=Q+jJwUl);0fhiL_rx7Q2y2my}FKYef2ik5=K~s9Gv6$Z7 za9tx+4afD^>Lus}DYG0Bnm3F)^-9`~4}BKOTph-RxLhQSw}H|Abz{h#{lMB)m+2CE zF71Ywzl?UP`|#RTYryzwe`C$z9GVNy@toT#%L^k@9T5JP3k0N|=diy*JEHdO?{)Nt zkAQvtxvw@Tq>J8MOi^;F^%aLS%j{|$I1$mD_9mvLjN0MbiGid^%Mo(9vnk1S{(I@+ z6^oG5Vq(cQMlD&ow*n*ju5p}MQB7>NdNw6+B*&9l?LUCur)r!B+(9cAu3sI?N27TO zO>G2>4y?J_O%u7@T&HTxF1)5n+%G`!TSE1nQB&8e@8OKQ@9kAHe)$b7p>{Jv^+p=E=esw!#Q1NyYYgQ8@Tt_xW(^L&_1eVj5x=`=Z@fd zJrmw-pz=%nPCX$buuYLB_`|VK?t^g^&l}P?dhH=aE*?w94gTQ7CV;BV^qg0lS(D3D zS<^Pyj@vz&xWiDHQQS7OPY&~TSz=I|n{$cZTcgfT##S%`qIShE+ww5|kE>w3-6TuBv! zK~63TbL+XI;Ii)mbs^rfH8MlXZrtlaaNe1uFxvh|-Ia?p8e9jY^O}j}zg0GOTeqE4 z@O2|%{1{3f1t)20QM^SLZ%-4sOwJdc)Y`~%JZ(L!R~yTQxKR!+mXgw5$h0Ol_G|pq zuQ;Avo02NdvA*w+I^p;a#H52?dYSNUj1-g{-Ri@oekD16ht|7@_V zlVsL(x^j^B(!9U6|D@aNgGfhrUd`ZH`72XhFnhxYH&#f{NrzN%Tzu8#F(gB!fbk&7 zk;%yxLJIXv-ntuN@Se;z>wVU-9@WOtXKsKsS5Zr*@SlFv$U>p0AUnN#dNr8Fl<5zA}lMK5^Q0N0B3W?zTcXoD>1Ajk|Ed{+{O zY3YIg;I?9?niNwaS2TQu#k}Sc9;sg@D20QqRBKgzVR?L5jC#l=+bPaPu~4WbijuEQ zfnlmw*77(Sw3jB5-i^dwRX4T7Ec1xZpyS*;me6djMWXs(WLDSMbwhDZ?B;ofjClc- zgTTZ6b@c}oLPJ*rB(ey7J!PS+iD9*Yg-W%AU%)-R^mZ6`mZgsU<;h# z4xt`6z{>8hc#bVGb6_Ym2ZN7kA)QvedohhKwf^fM7jwcim2KUlyFoa0fYRMXf@!qG zH?(r9p28qT{==TC`o0CvuR8+TEi!T41Yr%ohFJ>K^4Q8HvRwPowHe*LA(1=33ijWx z6J-B!wF8PT3Y?t%WGquS5%|HUv~mWa3)EjU=WVh2J}@Pbz=>s|8SJGU6y_3fwa9e| zi%S+fpI$%6b-M`jP2u%#eaT3A?I|r|A4RS!lpiNhWz*o$L|Z4E;6$CGMQe;}yk-!k z_Yo~r-)gm4eCb=voYbzW$s^5DHc4ln-te4G@v#QV$%i|Ux7Mu>-cb2^GwlbKCMS)c z&0lVifgBme3==*MCrgZL;wnNJHoP7xpX)mfE!hvgoC}cl%_BcpFkTqmDjyx`(Ffaj zUfs{&N$bN8uD}QL@w4k*I`5}*>3H~eF~)}U(H8y%fk*{E$YfW$##Y{iuGR72lhU?t z3r&qLNpqF%!b;Ut>5j3O^Mh?8bKBp(d}qNL#J3ndG&PS!-P{oemf*XE{edKPtZDD& z_LEW~*%xq@fq2wY8S6Mk@;Qweh49@|tDj$nuRnl7q*kR^7C7>t&unT>XG{Ons*fCvZTisPpT;ii5;iEZ8s_ol!V-Y1ULRmUe&N%I|Qf0uBBUM@iZmKH? zDhXpw#9R)dlqz&$0RFcJX{n}T=3IGOZLytPw*rgpny%f!{+1xwc-jN5o!{Al8HD}f zo0+fdw#~!i*s-RH1X~^^4oqUH2C5UO=I;p2yRdk^2buOK%v2^#`zxUIpn(9=W`;P$ z)xwuUC5{R<+XAWffR0Z2W2x>xFV~7%kyZ%6!F-&!GfgwdX7W$cJP?I>!e!^QW2B8k z5C^E@_{Unx+_)?9c*}~9c%6*W*E!cjDE74$XahSZUOZa*#Q^k<2M%1pmd}LiOE~x< z`8nAO)~^v)#zai?vi*2*HUaEBx@<3ozf)#x+*-dUYbReUn4fzl!aWwpRf|-otFRjE zXpu-vPO9teXJFa-HorfsE`RkPEbY3Cj2*r@LeSZBT0(&^Mn9>fxol&x67dm6O#s7O zoCm>-;ldUFLaO#~tve4NEo`XO_CTuUf8>Nwpk$aN@Ots3JjIyIL0C zB7j~E>}8x%dJ{Rq;dCr(O|uB%^-zrkL?BRO+ft&)m-KEM1BXT{6j8w{uL*pIDIHg?G> z_WQ>QyVdwOs!#B~0Eg1Tl3y^lhnqct@Jy0_=W_L`)sb0euroKiX?ZQK@K*md$6b{p za(nBFx$vxDL@uT7#IJtAy0f$82zEfT`X|716nWb#(!AvlQxUEfjP*%ZII?~BIS4>` zh1v{k12xWE$PsnEIdH5~f`(SNk@a=>o+w>?@o+&2r`a97P!5Wg%u5YUjf}F1ZwwlN z5rk)7L2Q~C*IF>l8b$Hi_$b~ye{s6$nyXa5tbHRSN!&7amM*~6F`D_B%BJPxSCznCLPwNSpnxqe5Ymk4_1PMgcyzi=@c!zEC z+Lwr)b<29~^K=Z+siTv{KzB#(pfD_{>S!TDaV$5S>9aIgf&()@3 z`qS4bx&GW0WWNGxo$}9SVtb+>rLxl(xPB&wpbb-UgquSP<*0@Ja}VnpkT_hhZvB^P zeBBO#Io~N+E_gyiTy|l%0^bA!xwZ8S;UW&UeAA#OjD;$U2!dy1;yEaR>kQIQUN&vK zx~1091ws*a8C2!Q_8SU1Wq!h4?wgHZdzcKMQQXjSc`R?&7TX z$#(sO;{fW<@)%M_k4L71)9UdFQB`t7c0F>4K1BT0e6(00)m=ZYkL!SID0-E?!WkcP zK%v1zK1-76AYQq-=F&Wjx?rVM?R;L(F2qsakiAds5SKKW&!etOUmZ6zH|`a5l5cK! z76f6SqaYXEsf2SJM#-|;3W=g;fhs8Hyf}Rpfgo)f8d)!FwWjyHAiSYImq!lDrNUMW zEM;i*26C*Q{xpXC)TnL*>#Xg3M$Xk179J|Gc!C*cp^Z&Jv?s3#D?u8XkzJ@xU_pk6 zYaXovlXJf;HeC@rMeQD@{c>=}XUVHQ~GH~lj42+6)ru8TKySKFsOFKLr&CSx@; zHjWcGwKgxqYWUs+^`+czxv{cqfu_&I^;wqG2&`2`*A{YV{XGdQqh-_Q^J^L7cKaw$m2a^ku(YYzU z(w&4?2G*P*`X=@ae}!iRN+-}I1Hq3O*5BaCw5pWFmFf9eufK!?BHzhf7aRXMCPnq;u|4Y>A!36q4-##HM(mHogaibp@`sgZrFA zRWl{f5(BR`3PRRmDUX`vb%AM~l;&GIb=3T^_y*`qc64r$m_lO>O*`a+%foe4_BM1O zmOYpzdv|lj>eG>2_ti91L%iq}3Cif_6Zg|~=?J#dtRfX8@;kh}MOaUj+ zk)YX>+{J|~4dpmeo+I+oIj&K>Gb8K9qs#_dyhNV*$6<9)sZH z&lz^7KhvXy$5aHuv@-1*G$gtUXo8m;C%#f^yal?ITLPqpG*08Q5fn6=emYpVhll48 zX;tJ+Prk7$gT6Q)MUL)Y>lIq}Q4V|!a)j?RNs%S|i1)EyN{bm3ZZjAzEVfc0doYT* zjh5S-=Qpp?rL&$n3YSP{ZgMcu>MBNd_7Ve!iPypMolK?X$4TA$CVKo0)NsVZ^Sws$ zRqh%5oTmK3eW>fao)kl*w;S;^5s9vdWS>zT)NYDR(>-%Bz^6$J|K0`cA%`&G5Z58) zYPS}gzqOl)=7OF@0@v(R>-WI6V!;haG0Mp;<`bXCf4vSm^5J( zFA#0EU)rwDg`H&ZvfjLen?4;~U1Z@a+gUYq3N5UTCe@TgX1vvm|TaM_AON|p>BDqIzN}vua^J_=*CjWE90FcC4SUY z67LMHy9T`m@m~>LebDcZ<`lmlP(Mdsd z{h;&H#+`9nP9>~4hY?wU+`pM}a3ZNhY_c4fkW3-Ubxt-nl@^5-Vwf&e?e zsI%frJ7N_BP->vY+4yNY!KBdh$pu5iaZJ@l|1+>w(P8+;do z55jW#QWqVYi;#QhkxRBJzG>A%x9UA$RUJ zBs2CBP~cS2Eawf2`V|h3zxc~>)LVav3Oh>ssdCYSZ5%5p=)eK?*_qtrcQ@PK-@E9x zHv^=vp~6I(CFadxJvta>so9Y8!>hv!LW__S#OXBM~>q%jV7;eVBIw%=s9W z@7L9Zfv}@_>U~z^SAB2}t?T8Fx|7CIb(=q~5W5K)!lk5L51OQvP)zVe4BxqM zO2_%Nx_i$EOX`uMZm{~QU0GQ%{M#gc{l%?l1#-x>7q-~}pDSBi-rD8`->MIOD{31O z>zw>lw9OuPYB1tdr#zCkV>QRqI=NLEvf8uFzJB&y7)AEf=S0gLQ18dQYV3NW-wTy$ z{{-fE5yS?EyzmuERTK^iV$QqgAJ7IV70OSKSQO^E5ZW3d8r}|jLFdYCPjA$`N!xc` zHJkXi;jtQ9)15O*wGiVyx>5e+p;s-MwM$#IIzTKDIWUt-T zSvDRvw)Ed+yYZgfbzKE+N8j#viM`qM?@JCtI(PB)x>Hd~&cJ+x+$L3x%nIeciM7PC zy#!e5%wG_#UIeTnN|!46O^VgyjUDc+qQbuGO&J?B`%dETEDB{VyE(qpvb3N(uXjW} zlS*kcp~p|c6_T0&!jh)|c6cxBqff3@U0Xr){#?SHgYj2o@qljvB!n(%B=()&z~U?V zy+Ip@0{~jr4}#qj_ll3{-L`O71>4Y#K)b>CF?*DcIhGT`1t*@^FVAO}7oS#1JDnaM z$oFTIr{)hD{(Ou9n;{;DB`u^QZ;!icbpUsvZC-qg()CwAVwT`Ge`3rhIlHdvx0y8< zd02IahzEA$51r^s7r42RI_JqB{ql@lcyH_qoKcwcStC_P+{g@QGVyNnk{P2^uc`Yp zTCX+~$~V6(_}{q0TZ+ya@yX{O1cRIAy?jyAM-((9!+9p84>aU>-~<+$NGIoyC;M-H z@?uO8#WulHh@C@H1Y^h{%w37pX;M>SGd{}_nvIb$%JKZp_)TWvNWRWXlaZ}yg207L z=-)6{Z$?=nH;qRZj^i4|QCcjq7s`}6>1X~6-@-6KhP;_B+L&=1&ocPjcC|Ac zB+~RVO`dDKMIym3Q;CVGr1x5j$kNEl^o%_WN6LvVMk=vCmOAvj!$$QCxu%rnk3|kd z0d}q}+!|6?)WAX#g*JJ6N!9Dz^ie+tuW_6XztL{9HZ}$2noMOtX+OxcGONlWuCi2g zBm)IQw8iqTg%a8CFC-N;6QKF;dxkCJTq6ycO`0yc3UnB@32^DR7S28xQ720#aPjdB znLnDm$i%PB#t1ULBCISFbDT|dy17-N)8F$L(ui;K2Nu7ZHNmSFVd}hjW1|{1HWA*& zUA8QdwSS#YcXFAtR=SHGBpb->&m9(98y+C*>5ClEf8agL*k^ zArlky8?GGzVTr&c%#BV|$r#Ul3%9ZC9DrZo35DRAJ@M1J*t2^Ik3lA7$$ z0He~-BnfWD6Ejpna4m)x6(iYUwghODb&7LTKvJ@Q z5oL~P&v2a^AvSY+2Cv^gjm(7El*G6|Hvr&2H{xKNMqM+uM*drK|FB6slA!(!!!|^9 z62>q?LXU9qL#MOxe6;1NRfa^e>g_wpRDxeoQZ`MH1^HWJDvp%GBhT8FrpMFT_mYm( z-?nxQX3RhaJCokSg4cRC1(R93qGDU&?cXA8Z@$Fn+-?}316{ER>dLNBQIx4RMAqtN zbG#+jLyMX&2&yBSjYKU_#SIrH8POi27&b*?nj<$kpfqxEnEdO6L*2q&F=DYD&Wch( zr!0ZBwp{CFHBoHzry}YjO+=b?^$LCiPNWphwct@VA?j;-;Z7`d~w`izr4|6D`ShEgmyx4I))`diaN+fH{<{9@>X5J z#TLClTGZ6J&bvM~MaZ|L7EC7p4M$2vHVCv&kO}$s_MN1m#IruLLdGYUDSb~hN} zdy|Hn1L*s;VHlyf<2;(#)H_FhyTUoSz2{2VS$%@GvoG13N_j1w#MnM@eU=xq22zHd7}|lO^-N1W%x8$!p=3iqVD0+29C?#c2AvBC@S2`IV>^IaIno)?N-E)iVLTRduBboNfIiE&dp4nm>3Mcpl zW?@*fR+2P-(jq}y#zk)UcTjDRG+W#6?WB~x7kcf9qxw?ittSlLow+;4nPfQkgg_yR z2G2)Am%4SP2NsBYGp_}RRRU4cFD?ZVQ6L}{| z=m(da6GL;2;W$P$!&XU^v#gIoBplY?j`Ps>-8Cy(1t8CKrZ~$s;zrKQ>T2A}mh6v4 z;zCsAw_}j{oS5NnH|B-xl7TBRB{nLmZY3y8#K)mgg=P`q%#3TPie`BBT&oO8aYCs% zYhw)^D^9rw-92ACA1A$DJ+HX;(z&+E#%!7w&)D|5{C(i&jDH9Xy7U7iI88ktF+rCN zqkongtOMeWV0&lC5;+&jZj z_=jMR*Oo$R4!Ieja2ah{-|T+PXZZJ^^*6_!4R-I7=E3m#no2@}X_t+E17Nu+0{3-u5-e zmK;c?-S@XT&Dpe*l?^7Qq4qf>3_yuXz%C-bZH=yvIi>I9jtR0Q3R-r4_#valhktY7 zAB9J`?+-tU!(Syn@ddLnM;(okn5SUZg9cMSxstz=by2SGo`fwccxPX{C^6T^#fZc@ z*Kx}L6M6#Gm5zPFOK{m`7uT(yPwu|6P?U>dmQ#GQwsfj3aBwS|PP?y5-*u`#;pVcW zS)$b{ehJx}dn;_o%4i!FDT@X(%#7O8;+@>FgJdgX#&$R>`%(5im%Qsd^3%7}oo0WS z#+lU&{z{ul*Ih`!`8ZKDftu#`n}(!3b!o!F=y>2}M6EO>=U#&D%go{Rn19k&WrVOm zKgnq-^Teq`C@0qTqSNyP_j~TeE859LVXgEP3|g=D=WlrRhxA6u&pYvNKy&QB5kh3& znjWauevSC22q@^0F(+&ta^9I;O}*4EHYIZ%+n4)tg0BeZ!RLYN^XXv=12+tM2s0T6 z^=9zPC=fMj{S{4CRbJ;1=Va%z^HW2f$^Mm%R(`+z%|Cb_X%eB39KQyqnN2mBX*+*s zru+2Nv>T|I*BwF~>VcTF*!VGYHlBEn_OA!hy~4OJ_tUq|Z|3c;+)dm>5HrJ89_#jf?Dovzb0p~6v6}uZIdyIt3feQ8{*7*$ z%$l_twc7TY*BaN_q*|ofxtidHPg`H>OZqLnjv_WK2MyDbyNM3EV(ss-%-yZKVI4n@ zz-mHzuXbBq9r3S~gdGIaWt}YyzZ(%vpGX8II(9WxV|t`8PuP{9bg!<7km)}phJ_$$oP)4GA8$lZH zf>YU@`(%d;N28@Wg9oU8yAbPrFmfb&`n z1fAPLy+Wfxi$as~sxGUHZkd8ZLi0kyIfgS0v8^ZT$QKfx6Op8}9`(Ch4VR{w$iA@X z$5Pf;6T&6HHqqZLLjV^0qT{hZTUBBD)p&Q8CI2--WRrk5RyC;+M4>wRP`H<|CSzC zqYdIXwB@8AT2Aq3aYeO;vWU-DyVj44kCKm;kDTt2&(Y7t&;HNx&*5!rfAuGg%TfvS z<5Fkt;~kX;6+h$87f$%ak8Td`C#%jc?aqXEA9{;TxN#GsAHYgQyX(%G-$^*hL__pX zf3xsGI)}HGVH5Yygs#IKU&uRhikkBC`M+MoRE;gCRwIZ(nb_6=#v?I|ylZ;4Q zj@u9Jcw@$%E3MNGP^f{`SFF-Gaj%AVR$CD`*>6Y}VHZ1q@I0Zr;cZDRWP;zGOeo~HCJp1nc+9n>u z1-6hs)H=qH9uRHZw!>QX&IQprp2PX*o&L`0fiu_rLrD}yc`>+btay)kFdiLdiXzns zlRL#!8X+$}`sUE|!c>a%WFRQIuB{`i%0b+I@1FI9xdO0a9?)FJgu9)5S#Iv@67F1A z2_dO-!#C=Lvc`*599E5O=UC3CUTeQX2u9G4m`8=XS&QBTqevNNIO2lVkBx@jP4O*I z!0K($tHJ0m1Y#!(Pl~7Sv+}f|ZXu;Vcm<|bB3Mw|=mD3&@~?mts~YTzA&`tFA%MI7 z_8s?!KUiWfHjn6&)-$4&o#ZcJ5{vlo0$8V7Oyy!XeBPX2M{8LM>Y)ob; z(=BHbmcIARI&k-DpKecfWJnk)7DBFL!Ps(91az8kkcI- zl0B~79cRdl*mgEOjh4YDC%w5ov3cZbl1D!6Uta?^k6=QrWrzUPAwjw{SurIN}m%C%9g5h&I^FJ+L zvQ`0_fSH4!rYu5I7lSV<%k3B3Tc%pe@hS<8r{1Xj+`|!<VSEaR8^t!H6}`1@^c-xS_Z(~)DXNNl5BzJ;ju`$kh7Z9Fnh@$oKJz4e)4~qdSfR6! z*iLtVIaIDF?+slp{2R4=Fd$qxJ_qJpy*p(eE3ZK?Yr2Qi$>R`@Rw}0Ng`#POnwEy^ z8+wS-oq;APq`#nM_k z`U4{GUp?5Dzj~;?@JF-mpQskCgjAgpT8H&P&Th{btC z(#g4zT>`D|t%^)w+m}#{PnK7i#Wj5=-xSlLG4XX{-x7FN!p^e;Gu$3KR&==C$};z`=h zfQv{N+4hqmjd1*knevi3(&<9F>5LwB+Q1#4nXsj#Qy>LNJXeUq)^b1pb>-}94LgT! zGreKvp~HA4evNJ9XMift_IE?&wRK}s5nd1^(2xmesODY$N?KOvf~%DE<(49eu35pF ze`o>y;qQC>3(xlZDn^pAn2%rQF=dC|((IG_9sxOhkR64s=gKc@#~njoY^aMfu@; zd)Vj+0=j5b{rOMfZ*F3K@1=_Ra)m8ft!Y;w_RyYx|CL1{`L{`vbB{+o9d|}iSI!g7 z3S0$@PB-HV#D}TH*rSUQW0D^>Z*$L>eH=+amb*&Rk4aMGd>|qJ(H)VuhiAe%zb}!HJ(hkpLZ{W&tWwxq-dQKLeKZ?pr! zn<68$#h3P}siMBqxJ|;3^&r8;7NoN`;qd-J`@F1Y)EF`2@Ec_nrh~mwMR&9kJ~>}* zO%h!~Z_M#}T-a_b7r3T#;2kX%R)=H8ytiE(ULFdTT3OR_*R5RiPAQrx= zxd>jx>Cm=kEa?-9@1ZmUk^SmoygtofRB7u;FdQhLr#z{&@rON~!kIkN3ICr`=XQsd z54Is+?&ef;iq3ldz_Xu4Q2^%XB*QHuX`1NBOA6{p2dpT3y9j+qws3v&2lk&&Eg(S` zo7sEDuVBU_ZHJgAix(EY?faw9eEYM~BO3A+X8#5s=A?-yq@CZ-81+DO6Z_{Ne^h@{ z`^eW7s}AeR*ZDWQ=dh0we+#f7Y5jRD!rLqjlq`vNkOGwO-}*O<$mV+zq`1LrMR@V- zx7J@Fg#zu^2hZ9PS@Oq9U{}5?NO^Z4i8_$W($#`Kf5Y-6aQ1RfJ&(xm+bu3F&1axj zLGQpO;Jkk4F<_;<$_}+FhX-anTuj{iFg9$1)bX91?-TDTdw3iSadamvt`Ow(&{XB& zb!O&lH^AG<>ntDqI;g@Lf`U~Q^wkJOe=K_H9a7>q|{fe^?FgQYS&xv=bQmcV|%JM&`q;r{mO-E-6f9Igdy zaSaQ-0kd--huTfQbWf?@vT{~kyIZaHynXiEzT~iP-#OpdFPn8n!Poe? z2ZXzX7n!@6*V@uMn@1bh{QtIc$MsdUkJPd?m#}oR^)PjDw=_80fgu%{l_@G)`pqRk z8UMzfL*Z$Ev%6r8QQ>v}c>e-05*YoX`;*WZIw7}3_^{{Ohc|fZ)7yAQzO~5t^X&uu zjr6J3erI=Q_S4Iz-M{on_-0|S806}6S0o{!M@ov-mg)Fo|xFE%mulTvxaX95yL z4tS@4&-CryxMA7ds#X4a!GN6JZvmD8=>fq3Z&23IGlOU_dMGwAPqDdQRtcWT9B5Bx zY{C?fLfC6zwS%(zbm}%S!*^By!T@R>77rsY*<+ELriZ~YVIIEct=)&3y}t8C+dje| zZuo1onn|-pyNR;h#Bc+Olu;P{aXq}vFFwL;+|Q}IxcNUp4HwPf_S7{$XTs5>-y5t| zjpWKGlTg>fFY^P+u|#QtL7#mq!mH5x!I-VA&3b$*cIo&mk(Yrp!GqP?B!)4 z0H6xq5aXez_KLTml=P~!A^9P!^t!S@+d;f@wGPPlDm2c_#O@kD(-RS0(Km_k@A^Z6 zk)QQ@!BC&LH$8vAud^E!m9kRb==Jmoz<4aCYlLy^{gtU9!ukds|l55(EpKYqd0H*j63gQYhl zUFe;A3U#=k0h$}`XMP{J(7oINUkPCtauiGpn0*bZ7>WNI?(%+;d>sp-@KmHik2w4h zFF>0BFY%w`49s1#2Uk?!{jN_N#xSI(4pnUAPSBVGtZWcz@6ycy01V*XQNKveCg;Z@ z?41teWkJU|-4tEpH^Cj-4mPFzdL2-#rOo7>dx5l;ylQpv-2;zgq1$@TQn&F32zD*H zA=5;EPB7Cja=B#;K=;;i(b?e!f3di3_i{si5ggm>Cj!d5*fB*P^JDVX08K<(o+Lg_ zz?q0O7_N0hFn<8^HLWijSl9mANVh1rbnZL%py16u%@b6iN{RUMz{P~t4wQFXKSbnw zg!F5&KuM0pvd$Ks!~`L-C_TxgWK(QB`CG!&FBtIYKzFk!Kh1=+v*^zNP_JeM-TRC* z;GTZyqy4Q%@29+hsR}jUgI$HmJbJ=eg|i<(6ZEFJNYVE;c^bb;6eb;jj0^cF(iOH` zQCLqd(*5;6(VyItuL2)Ei^LiyRD2bY`^|+eUnU$P$tIg$^du$_(t#nX%sI%kSlROD zkv)Ajnz@B-+wyD`hqSe^$xP~HJaMJQuE2sjiwS3#oy?(TgBdIkj2GyIz8$T&3axMD}vN-e=oGw9vgs=#pb5mr(1^- zmWesPwo*qU=ea(pxuqA~Elc;i2|>L=&*D#{FiVbSbyxrX2GmZrJC)saMh%;CFtw`H z(2sn7O2L{fx1Mc}nm+RD7+&UNB2pboX46WY>+qpKfwR;uftxTt17J2#niku8Hf@X9 zrg$42KxE{ZnecKMwy~$sG z01rx{x=2GI<-`4#No6qcG2ZNbkY5x=A~t-xm0&F zPC~xXvtJrKj6fY2m zyDX|L!)Y}&`^awClJje)xlB%`*|MCAajC>L^+K>A(liyQ`CuyJyskypDGmCexm5cY zRs~%BO{`sEY~7vn%THq*-P-t@Ipf_xfg7XMNC>;LWL1UMrG-nyBfZQu;}z@b_Ag?YMD=5s{KY$mVc9`}eyITj zt+Dg$vpYg+Gm2=dL9SOSJh~(1y$;#x)f*W-APYs(){*g;g|}g7UPFcXW@UqnX;y#R z{=JPX@n5e0G(kQLY?Ha;OSb`tALA7wcEvMJEG0NF2iX1>N+i5M z(wMt3B_ZB%8ugdM3jelstd0y;yX>mF3EL#qoc3z$T%EhgCBXLOBEiSd5{eVDmhtte z)m596MRa!|e_jGR)BUii!d`-X4d^Qq?@3Pa{UpBoR~F+5DJqwTx$Ei z74xZH_4z4H&1-n2k5hA~!@oE=<((u<^*m@8ap$zwxdiKg{F&b1)jbb^oij^1j-2Y7 z@Q-><<4X*VE5H{YfIFBjU-Yph((YwZ6i`jR?jM3Gqk6fq=9L001tL%oW zdydm6+>aMZc}6yukQ~vq3S4S-VRyKFv*_8k4DlRiRA`)BV7PBmxIHz@@k%QtIi@=i zcw|v%KcRc-VpQ!xvqkB~T*rQGqE=kcN-TG@j;Y1hVlO#v$gsAyt7z{ulbA~6B((l_ z#9ifHvyCeYCAA-VV4(jbgywH8(hSNDj@}nNEI;aZt7r7Rk3D89YwLsN;Prs!(xsS+ zY(@TRH?_+dYtk}Z^*8BD?PmW9K@HrB$QA5!xTWZ-Of7fit-FpBV9SaxL3vD*lvDhV z`26^}co~Wn@near7y%4Vt~;~C!qJmQ*T?Kfw-3k10U&c(t`<1)HTZGjoV41%{&VY9 zc*TEhc6N4Ymn&Y^uapPMJ#KEaZ=;pi}>QOirK#!hqf&SD~xaYGB#3lpfyZ2n$;!{V{bh zeHmVL9+pl)r@002#;Wx{88F$6q=x0gP$Q!w2L+Tj$tPCFh zxx}=jXC))I{g%P~!HPlUaJg`9EK(}8Z_s1QCVZCPVt?f%5Tz4kW@WuQI_cd_A66W8 z9NH)FC%pXzs5pPm1m+T?0DtM`WW;v}K8jfR0+$xAl}X)IT*djNiG%n;+@enL7FcVH ztwy3^wE0t91Qu^)N34mj-HA>g+*{m^`QF^ot^_*o3#QAn8~JeEn9hB=?q8-?_}pFn z=8vweu7E(N&Y1I8vz2yTTvxfX4#4?w4S%QK(xb(SjIO<_@2YDV|KirNt&4y4+u;@F z)k3q9ir1wcDse4>T!I`g3DgI=A4%HZComP8oE z^B^VoR*^4ZwmGZE3vsZ}uaLL@p7{9VK@4*H6JJoQJ{VrjHU;@pY4wo=J~0+3V97#KLf@T z`XfBx_e>wR<9WxgaB;5>amzhwJAgj8L-q-G%N^D;hjNJl&A}4Z^Wz=$ZiDPo5_7ya zw7k!3o(C!hs)=Q`|KCg-_P?$o@X*9j3y{PpE-bTsc#xEkj8Stea9$9CQI0H-O#db6 zYx`0HNq{FDND8fmxA=z@`ClXCpm*-$J|8l0bnqYQ;cSsl!k$0hM6Ow92Sm`J1ySyH zM3ABXyGt@Ks#oj})PD{C51bn1MFubppNpWgo&JF9!29oeI8MV0IADJsI{sqF4_mwk z@*p9T*-#W0w%IWen*Y?J{6Vj>RzoNv2(x2F@afSaVAi>@BKb+XQE0)@B1u^ewz*Mp zN~~zv*En8`GAt}x*(TOuEcIUBcdR8sf3LX;!3@Ederi&XIF`k7rb_MHU*snX95Inkn?y_mn0`WT!vjbD zGt~bc^F5Y+Gc%SK+e*%@shOG2aloFC{W;R-i@oCiVM0G~?e{LDfPgTNhk(HRXF_+e zv^DkmXF^x;bol=p(QP!RoDMpij=Pg9O#JU5+*2jSRi9{{Xc`*@oy`PVe0&%@OO9<+ z$Y2vOe*J+%99v|ow7H}3Ozjf+j-FUcNCq+W{A}^yie>Z9j=qIj_WS| z4V4Zn>-UY-FW?qA-8QAjpw&zx=|WXz1<8s1yR{1Gu1+Fv!y!>y75xr3ry*134`Xc^ zK|025UEcDkwr&-*7<&N#PU4>;>8{7g2eYR#G$R@`;3cG+`FLxNkDE7lPoMs;ZuZiP z=wHnx*nGxKXJ_z&($Q;_2QJdQWQCk?{~@z zTm@hG74*0zSD2+&kk4GUB=5S*MloT85w#OchRwXU$}XkPhS}|gf1S>|=5$~iD2^8k{TrJlS+RX}mc z>>VlC`Jk$b4}XuEqi2<{d``(F{*Xxb6&Wj2Yv?Zl z*mitInciNc$}0>N+IzNMBve|`4wo7X5?&Ry+mPca0!iGF1P54J^b?S~C$!ZN&98PC zrII7oIc?>x7K}~d@6ISZiH}6{L^jELN9{LC*V@cWRJfZ zFWP1P^&}=J4`pXMH$9px!>$^&AaW9@TRk;sfeDMi^ROwS{~U@&HPL8(bbg_K>FfBke!oxH-??cO zRL{kvnrXq2@qB;5$yzho%YM=E?D!+#Fre9^Dw9IBV5@M#6|I&KHhExMr1(c#iCkTS z=DV!4gfm?{TPT(o z9FdyGk0lDoOuLKr=qoXq5htQ^^{BOmxB-SOtk1gEN26!kaj4@Zrs#0V>eIgucwL3+tZD^#8r41 zqcb zW}LkrM8}ym*xphbh^{mFcVsO)h7VSEhD+m8BNEH z^Q1WOLlK}wus&|`e?;+95oGcy1(g{nFSwJhHvbW;9!ba^yz>afSZMH1Mgc~RVzSmxhiiY)GnCR%SFkHtaadL z4?yK&PG`92*KWMzLXrC=CB!jKiP^ZDKswIVxYFu{KN)VmAE)NXd3tki?j4>&{g{My zpmqG5WZ~(+NJL<~A3KrH3F+~l7_2S)0-?FW*c2qsi(3e{E;1tQ@Yhq|QI%v2Z%S|D zf|EQ+VtM)$7feIkVT5VH@gx=r{_AY(RiND+hB86ZmTcqB13WC&53N1Zq>x0+u-t_IcKgt1P=m# zyk64ThD!5L_Ry-lXJK1Rr6q(>Pz30*3oCbcYNzn*))a+aSfx;=14nlbjS^2jNl7Eu z)@96;@Ku{8QKp?{*;~jd)(*2bmpve@ohG!9LUI7R8F0WH=0Svh`9n;j1<*1lVwIe& zRB$|KO0-nS;sr=76B7lBiG`E|{j&R^hWWiq;*5ZZo4kKru-B5lTbn_60L2zo{z<5H3+*)y-Ql)HE0_^{xNKK zQ-!Q_9j6&0aA^XC)d~xxBNI`nuF-j-y(I^afJ(euopS}v3kFG-@dCcNXs z(!=5}1$$ve5`r~q0lzb34RtI;X8D=^-|`}d*5)Lc2rs4BNG#R&+0YE`fPfvl#=1Wr z6}2&xDp`sj_t-Mo(^D^|yis(*G0-v2rVGbcrp$?cK8;yu z2TM-QV8^0c!>d5P()ZFY|FYKiun$!EeD*GrE)`0jbX!yeh}MsQMee-{6N%t8mJEXZ z7Y2wkY$A3Bjrp6`S1{ZMQBTBM5iIH}Q1lC9MxXfAO4KL(_zL+0)Xwl_EPR6U0mgn( zfcxa?;XDxY$RsE#QpyQ0o)%L*CZ=}ULHAhI34VJ~pF=chcIi4lAplXl4tGoa^^Ve% zWM*I%^nh^2`cP?|tD+NTok?^Y5YFk-Fb-C+-fed1$L5o=QFa;=n*c-dublC8ZsKldF+`#Yp|!!(C6Mah+30ohR2GGydw zJX~|ZIV`?m(6I7tbTwN{bKfMh(&Yw0>1D}+HWu`t0(hhBoa+26(BH&YMJM-Ij{#*l zEb@47jiw-`9i!L6?~+&9*fa{^;cj_=g0r&tI1Q$*4Dlf-MrbK+W&vzrO;R}br(1_u zH1`-rkW@a7Q~ms4Cn7U8bg|S&96n$FFNVBTkQa(w<{Vi3Y(&+@xD~4dccuq-2KGKL zHmZu>xj5!9di>VOvRW9G?p5r3oeFtgL|jRgKDMZ{Se^z`?qR^GVT5Fx1_u#Y?!BzI zf)+Kf*Ft^*5B+UH^A&qSKW4gu(Fn2t^1xz8m%~$+kg6GwOES{SsP;HNZPO#XA&pR! z^Mf`b!NQ~$KP(du6;X~{>yOE}S?cby#0l%h4!uCW?oaBEQkRp`#jj#CERDg1WA~4i z-yiZ6@{gln)~kw~tz8s%G>}srbw?cvsF|PS zB0Mfn(xdX9e|EO^Y9~3rx5H8Dxua&P(28aCf!X~4Vr*+x0J{iPYSBNHI>%ZWv4f1+ zUVIhMb~|)2?1{G6{oOegnvUe-yg3@zuP$U%k|qjp9i*2reMJ#9;obg0t(Ag|zk#J! z#o^Q8-IcQ?`gIGjRrYXbN8>3_&f(+J{p5owriAImw6liZYViW1ralbc4N!oP#x$N0 zJ%=A7VHL0okJ`cs*Cpzq>0aBJLx&kenYU76_8vTr9YRP{sD-8R)YIhe;9!z#9$E|j ztzs;nno`|EnHqSF@C;c8l4T(z-h_H#h$am)?nmb*_!V7mfY74SZOk-ugPQ3CLC82b zWmuiH&hHMQjx!g?cT6z@OIKwPXY-yg=8t+p8d z9cE?ep08Ux)nUrNB%u{ zF?;zYUFpgCH5XYa`hgi6Fz*pvOh))aDy&J!xQi7=S+I@T18rYqy96Xo{isznyBk-= zY1jW*!Y@1Gk+w(VJiGr)UOXmdS32}N@bKKn=FKF!VcVr1%@RqceI_#+acJ8X>92aG zb&CQ{R38C@IyT<3lIr^In3of)jk+|(EBe~l-Ug-?{##)Tax@gVn7ZC?OZGB;hLypE3esyzpu(S6jW4TGY*+>#b6dfO?+tx~#- zyI#_(_c|(Qpn#xzkn_&)~NOt@x zc5ebG=#2YnEU~l~ea}_yEM*ckfgksv{2+3+=_n?EMlC+2u%|V0rTaseLm$twhU-99 ztiuqf4sO{V`r2)UhwW&Dud3-#YdwhRo4Exp<3S|zb_@&S4(6m7n=!77Nl!|uMOP&p zVOHNs+u^IwGpv`M|9Hxh)FXgVz{_%wyIpSC(aG(zpB)d2r*g@JW`O_xL5#_gD~mFU zQ{Nx?ZV?SBQQPjdP99TjwpWk2R@(NmPY7Rtkmv*i$# z;>O^LXSivLuf2XIg?)kKQ51D2KE;39Z0o|SG4$qYvcY{B&N{W=u(f;UHsme~#S8qB zVw$_Ny#c;t&Rn%+*2V8OU$}2A;A!wl)R^f%;blnZAsGInY{}Bt`Fm<Iv0u7A|ue>6c{hv$TIERM2FnAWp@t# zY!>KsetR^B;6_zTcZYveoGS7o-Y4pM^m~e13U<+&$fWMAwBoFU{q-*1lQD(lIDo`_ z9OKZ(e}UE1{)N65MKT3G;UcFVOy&7+VtL7F$Za>J%$H|UmEhOTIip}IV#l1XT1a(~ zUNiCQ>Q0-sz6hNLGUhYM)mCcFo@>^(!t}d;CR#_EZni8OK6N4iZGQjfj>DQMbb>a@ zxz+xMB27K*lWwyo#FYzhznKjW`io73YKRQ1;IsWlIUKBfY41JN$*tz zyvF!BY(=ydwL`kZNWT)E=zu;|L2`sQ=7J^C!5C43@!}yw(Gyc<*waq`?7H8|2a`hg z^g*&0^+AvIR37~L)mr|Yzw~rHt1Y8#GJ~w{ibbd%ZAB=a7hcL23pIpjSgSW6z@*lQ zIZleWAURky5r&sQxfFI-feK1YRcAC03Cf+43Z9x+S=3U9N`1&(8?VUSIB27DA7$l|IZpih^~QfT-;ie%2#&P zD1=UM!~)5s&9xmy`E^D_Kr%i2g0zD4yu?|=lwHFzm7x>h#2D=Ej*ZqY8u*u0iM>22@H3mk*j{6??OU z+SmdoO5pPHM>`5PIZ^h)TE)zx3FrY4qVb($|H-gukSZmDR4^L5?ME(YIPjH9}`D>w0 z;r`ycEIbnoQVRK_7W$F)n?dO$B+;(_BpJhBRjWcOFI#h_KUJO1JY0BVwS33&8|#LX zX!zcQ)&?eo2mP@X*ED*xX0a8ukU*;6Ppj9+->w$3MouitZgj(>&<=kle2nZF?u0b7 zAzZn(Z5CLW=(E^|p-zP;)_(7naxWIPt3eT<3E3z?>(%qjvJ26kKeAbpm*j>E9x%fq z%HtqP;ULQ4Aj;q%D&Ry);%p?b;-s*)b`4onGhcGyq_g70vf`w&wob^wA}!&=|Y;qKi65|i4SGE33(yO+U- z5wZKievFAb8SMq&I^INCz zMH@NEC)qI9gQiZW@lJm9Eh}c}I1cC6J4$3pn&c-b>F~8Pdx~B3~5aPu`vvSg~cVmAAQ|-f~Go^FUiE+$U&(-^yfvJ zLhl*we4E35ceoIJ7b@hvp8o8YTq-~CO7Lq{N!6lhCYPvhGMCHhn@a14`F_?WLU;2a zp?B*YgyJ8O5n0YM23;nPK?|VI<^oD-ZLX#Y znevAUW%F<%_TYwyQ}(F$fP~=e*3veST{6L`;7lO*$C${*#;9WIim;9Ngs5Fs8q6_` zQVFqXg7ji0BQHWvi{-r3!llFSkB}e^=SEl;M1ZCewFXzU71un(Ni)6QD1EA6g6FSa zy)8r`u^XR6&R^7#m9k|;EzvKm!g^>YVucO9;uDUS^S}l8c?a$i4U5HvL>F)3Z9CsE z`VCKHCzJV6){Pgk&?gd{PFNF@hiI7o0|A2C9`R)|{;EUyNF4fy2>6Pxcxrt$54qGv z@ws)|pV#<)UCtfn0@$^$9u*vMt$lw&)Y)*B4gic%{!!W-gRgo{ZHUjsa?j_gTpE;( zT49C$?Uu9U$?4HrcCE~sDRgSK?GQ!)l~i;UA>{`xS*gZ+s%~w`v+O`YJ6urAANuhD z)~p0uu}m{2%Zemwm4!lFl#enemmEjM;Q1?J5dfV=f+*mxH0IqY(UQe~m(+8sZaTFa zZK63zvuV~c1{4h#ZNLc<%KV4+T)3;$(L0gpx#C4n3(g99v^>V^xC*i7762EuFR(ZI zEqI6LuhVj&rMr3{xj(s$S}8h)|Krks<`dIpl0H?&M7ZTpVCBuS0W+%x3+(ims4B=1 zy#jlIglc0ktrQGl82JefY4i$MtBs->$&VndgHFNlAC?ufuo+)rdd_u%cP9ne!8LkH zXps7*zmEp>?0I4NiOOHTE|-&jO5Kh&lh60LLb_Itiz{t+Y6}~NSe-nM&)m0SR2L7c z*@C)~peybOd`QZ@n)(GBVhYBOh!~iu2zgHZS9jj-UGDK-5xD(ZVZY6QvBdU~=R&oN zsBR!{xE+!?r94HPVwg{e-s19xZ%mlDl04&eBtJ1f5FMOGFN=4s5}9%s0*PWGT)o4ne* z!hXPc1$B!0OhrcwbEfET3+wD_33kc&2>OU;_03(=>iC`6=ka4UL6f~}zkde`#7)T# z$=txVV+i!AGEI!5n9*$pH0>tcSUqY#%e}(arL0W0_q0zqF&CvEjJqcHdYt!q-Pk`_ zwjx{^SG&pO{wTOjx=m_rBh9;{5)sv@#I;rAe-#($>{NAcmHt9#soF;sP~rEA-%4q9 z*tIKAn(f#XJRz%5@lf=|1|;;Zcx0%3CN=AM=#(PS48Y{mZ*UsMWhxrAbY~=cXMcFHq^M*h^*9{lQzcNn0&?QPS_gi zn?2P?EZ3c*KVnb-^=$u38bEibYsR@`vZZ>0v88#!aE<1f^cv7TKxVkJ{Y>)2(C?nw zw|Z9h=IV;s8gRWPye@i?c}EmNTNm~*tY_43SQTrj3taUACtRvbT3;c%8IT!=IW5x($6Yjn4X#r zeo#1GI{PZ#st_pu()r4O!2SW9WtdI0rf5{mD2mv{QqJ-|a*f%i)Q8=NPGZW6d&M5b zD9e<2$Oq(>jL%zZ*{+(b+Hf0i+rGA)cmyTyOg3|n zP#8;#JB!N_4l0=WuUHWnO|oOAjg2GUU5%ejtc`;veAu#>Ts56A zvP?tBmY5fmO#uzGP1d#VBe$&ueY1d<`CHd9*Tk%fzYAuFe_;MdXj1%E6EUKR= zpK+hvEhbg?G!oPkG4l(3zAq(N)vc@fm|g8i!nlUB+Do zT}Bp7i!Q`LgC^?XzO6M6u z+-BT|Tm;VbJnG!+oGzS?+?&937tg)s9kg`$qyLU9j!4Z@>K0mhU&?_=cfNNlM`A}{ z`k4b4M5lx1!d7Y)3R8LKhGvYG#Fk7)8YN+K*rk z=i}`|f<=af=ELVs|NaY$@%y`L zcFT&KgOidlZ?rlBw?3!J9$13$h#_?s{LGi01`7X3-jMMQ!BieT(TR4CwHmx@{?#%k?Kw=mn3r*kbSg?QL*Rs$K_p%$zl$iNqphC)1ilZ&o4Xj2av_oBqx8#uP zY+drN6=DzM4f1mdl4Y`Ins)i;zZrq%+bDyh>1KB)WY0e~r4Wrs&WRf%u@$n4G#DYz z%n1wz8DUmjpPkReI}@3Hqk<9z1}YA4ORA1_`{`bdT@ZngD}e~I5N#P={u_bhwahgk zT7~_mBD7LnLSVBg6`mRNd(>9>WVqOE*c5vd?_36^04%C^B{x)L;izePS~Xth&g*JM zIZE_HIA>u|w}>z0YXnz>Zvi{P7+-8(bUqnhn%U@AwkUWZ5q9+zrn>|1~7nJ3Qgo{4n_Y{;gVv`>yo`1gfg zF*Ldf2CzuHIo33u1$Mr9b@_FUCaA16C(Prm_Yf4`wOqCM-;-H#+!5bV z^Y?dvIkxGVp_X#1{;+h)U-S%(jE!@xOSw8A6#uj(>@t0G5zPjLM;q4N`V`*!1ix-I zBPb1n2?k-_Df4t+bj#oAS2p*Kz99Zj%S%J}_zE4tzZ!-=|GF3cSzcTLwkCG2!2h#) z;m5z)h5woVt9l_`N7tHU$(f(@K=}A-)VCL(H%i`b(SEe`>i{N%=XU6uuiNYM zukEa{|61(+<9(BP_@*Dbas85q>M(mle(Ap5rb2n%ct^IB6=;-hHx;mjX(+%jV!t;i zm#-XLn>@Kb&B2Wp1tYQpQn6;GN>juNKN%)IZ;aWVG|%KS;YH1yS?pa9aWJXJXv%Jc zafP3z@AgjdV4O=Z9P;!3CU%KEe~NESxLwJopZrZ*F*%o2Tm&?^@@E?fSeyH;08;f- zF9WJ(C%6OjD1`PO-GJQ@XhbmkCKuPyvdd1`-kU8ugw%IOoz$CUxrakq~YgPLiIecK=QKEC@ zG{jis^!6imWHwBEqio35`6~_6h%;R*+=dXyqi8AU3}cC2jNi|KCgN}zX@ehfeBh3i zo*aRBs>tnIp=&=nZ7G6_S&U6m4i^r!Z0V=sZ^n1IipF=fJR2$pgF|M4nbU?)E_z1@ zTwiL}yLPbG3!GIP4>F3al2__^E&Pq!JeN~Ztj%WclFcem`@IbT*XS^cZ)+Wht+1Dj z{5Z%Sue6L`oRD+qtK#?gvwm%w%Xb6*+||Nhl!-Vrp{kBZ>cW5@w-B!e61@We{N-%G zWKPv+ggfd#FfyKA4|#xm$$$Tz=g!FpF3@9N*jA(5jb^r1&)M|zz;r>VKG|T)u>j&y z%Ma<4Bc|~vQVAD8E- zbI|mcMFwF&aG7>^k!jMzu#%(~1azf7e{>Hqs%M;B2~@{kC9`k$CKOUTEg$hfPVdfY zDbm|Ql(lOH*E1eJ$O^k7q(jag@^cTUqS|BhW`#`i3hzVD91UQNAgG*?;u`JZuT(%; zG*e(aa-=SMCJNX?lNqx{y=Grw zu%Zs^HVsy7S8bY{;zmd>6Tl1XR*EYDE;5hXy6OO!M*QLnk-GmTYR{XFYL)b%;MFM4 zg=%02xY1m*Lu}oDvEwZZhOGHYqf);qS_$H_IiRNVY}e<6(saRR{B)9;t`NlBuu0+B zpDk0|@Lk22N0?suYx`~00;0}y4%jyxriGVpl+Dg=^F58ikL!WJ!yS2_r>8%?XhTSv z(yXUMN-!FQ!mq);2I40?ydj2efulpZKAn0_GfV+uaZ}IAx1;~GK-2ub{*ZVE_F+>w zNmU3wCOf)_t3ceQRorR4fW-0Dw`t_fP^(@~^IvO4#lDLn^3!oK<#xN86j*e!w5ztl zs1&6nGmSO!zL3moF4_LJMqx9T!;o>4ym&2z5YAfj{1cc+qSBBR(2wy`vbxJ_s&Y-)Dsc*D9D0AHwfQenAe7Tog#CfM}{G0ICobHn`HqA4~=k{ye_$ zg#@nIN(~pL@AAPHySSzXKqNB1fxQ|jnIEZ z(+c$$-QTk$7PT0yN@6heNG2cbkpg-~Vzg>pQu5qGK~F2*!&cxeED%4+DG;Q3VOK>L0#ZzX8HQY zn|H{v27eopKihV7Y;r!fwMC$~1fbSgMZe|G&&wSJ<+7`7JF38F-3w?QhOJS+fE@Rg zTASkyl_-H(=p)9b?zRfyK~{Wp{3~3fbYt5xON7D$Ru9WeW3Z1x#Unl*A9}0+Er5e^%T?fSgP;wc5HOz-y>JiKK6{l+#GGm4nT!Y{q()KWH$N zr$6oYrl*KGx}Bf17QT+A;p2R=O_m9fg3LrBF9Zzeg^8eCIki7a0>uU;i$HpvM4QHd z5Ii}TZ@s(Kmg>LPk9~iwKfF8~B;GZh>oiQj`zU+%%*w>tE~$kY8}$q=Hp$%Yth0y4 zP5J?MX>O4{-}yMiqRxFAF^Dr37thX*Z*@XSrOQ3Xm9G6t|4>xauoQ0rB>}rXUC4TU zm0sST?=3Dt@!p`3KNqqi3^o?ufgL()v?~i$=pm}zkz%J8c=)-{lYc^O)7PM z5O9tew}>?D4#w7Nyx$at#x0ZFQpYs>tyY&0$BRCJ9}@9ThCFTATii%^WHHH3Yqjm_mQX%21Lk| z%?JgM@>e9y*FiQShO#$Uh^;E*-$|v0J-My2N2Xb(q;R4|OPn(4EfI;v#Irzei2Q@G z2*~o_%-rb)EfRgvP|DZbOyp&}MSS2-TM`X0v)fcoxex!|LrTot5TAhiEOzS_ZDr<{ zirl`g5o#>DpUY=!SOm7>iV;dgBsd?x1Q0`j2bO#vk#j6!wQ~6$hWA!f0WYIIKx5N$ zprmIOE_bE1y)=*GNHA@Q+$bWcHyvvfnejc;WMg(Qc|?qH=qy1bQvFu+0#TqE%cw!i zT#O4V$xj>J#M_I!PluYuh%A_VK+YrA2&~ddAM6^&4{7+@2i{KZ03wMRmoqi=!z47q z)neiTH~{KN_eT*q8iE5srR+UJ`JGYSg|wTa4n*#-;2e(nU&+zX_Rnh5sf^jMNpoE} z-UP>-3*rZoK3pDwq`#Lg>+b9Te6cZuRzJM7Nvb@WPNKw*SU>Rxf=W19EXI;^aeh zFHV8s?=*JV7gEFxUUv8*I9;V|xy9PHirE zoJWiXN+1`r|2A7Ci69rBuH+09tyxcgG&*`No#mZ-5UrDlBxRcBJgil^Cnw73exh7u z0?{H{C!*|ejQsS-zV+nURpO?XG^5r(%8M;-BsNIVUg63pm6sLF|1F4?#UZr8X-^=W z5iR^&l9-Vp^qpHf$-}K_cLQdXD*xbuZb7 z?~#C)zmzMU_9a65mN6eamgk$qUX(j$zkz3+~c8(+w6LpJ4Zq6g(v2WAD zzwsSaR$DZ z-7qfY48DUb#hE2PEm?G+KXSajVDpHhR3m=dyR1#+ZcA6{N+jqxSf(F#-f5gp%YHffe$VMZ0sr|$8oBN$S<-sR>5v}b?K4oS^b8+m5YOI2JZ z)lR~mU4p?#9-(|?o=L^b0j?wL$0-o4VYUjz0-A@OSEW_%#kSm1ud&>XC65nWA}hwf(TAM;_u)(;aYdCLMMU zz%We~u_1hk*E_XBZkYA|m(hgg?w8nK$xEw;kvM<1cD}wZq3aTJvybYYQX`eqD5fGo zIzZmpZBlo#lK@M;)`LjPxaM%8*W)fuq2n$U85UJK*&6At(mUUk+0r`21bmOr;(a|v z-LUCxH9ck`A6zp})lAy47M!e4Gdnw6ojbzXrv_{m2N&dSSO1kx=C$VWDQ(=9p7vk##aAgz zldY6YZ2k+{nxKF9D=#qfIPpa91KcohvkF{_cAsl`E0)mH<&CFyJF6A~%Fk=)@mh*g zPCiSGk=rPm!&Z4`&xIdiI@w{*t2$l7dC=$jk=&w{UF#P(sWW#j>;>k}?F{bUO7{G< zG)Fdpt#VKGQp#syHlZnfUvQiAIoVjHlsd;I(e{6&lfSaF3sfg;)3TkatN5@F7;TJKJ!cBbLu%9bHlHMnm$WHu zYiwy2KcYx?VKn@FdCU`}8TPS7vN8ooxL5fB@G(vVfAh+kAqgt-?S~08C>UtGdZk^wZZxQnfxr4iUj1)8ACU%bBn&>7FG$sNjm_tdX{ z=a!Bz7H+?dwh-8=#*-9-vY1{{w{wp%IuigF!&x2#ol&kp(`Pooc~oC7}W zja?p^enaxQD3FEN5~)S?JSja~{wDKTTihzlw{p(PHB6~7ztOP-`@GyRloe43ynhy* zoFJr3nedA!=)0fV@vAB;tX!FO$tY}1cBXk@((lau+8(-Ac~*F?8V5VtnIfcg4Zp)` z{PCAIqXTjQ*?=TKlD1#0)ORuGzv=|?i>CCx7J>L|b8%gFn_q>WLgeG)g#P~;X8oE7 zUU&v35}K;o&Sc?t^6JQ1)V#2Sw34Drd#Knz=zedB|V9kVFLZ}i$j+MkEPP9FTUo#Ha=M&>W>xY z0%lu-_+4==uk*kppO;Jl&sT*kkIPpHf-(I!Y)^E0c=Rt~opAY3Gf!Cwg+cl-C1gPPCvyVvtx*M4i z_<-d7%l8QE%HkQvujshr)On4Q*=um!e^(s)5V{O+VL14NefgMVA@sOvpaJW=O&0i! zda6W?1hjB$)1;JXY<+I`sC<3ybQONlG)iAaT3vZxGn|}cRUK9x7MRpwwqzZ6cRoHe zR0-gid?s6bxxPFeto0wL#e2XijBb=~=(Ke_-mcnJdAVG;E=gOZSuHfYuF4;t7U=Wc zG?Yy=WVIJLpJ*g)ws(H?@2yBwe!U!A@x4}EUwqk(Zu;l7o^twbEwvnN-mPw!xZWX| zaG+6#kqM6cmK3Ih%%e>6r9&6(WraM5p+iG}M+^!cQT*KyNKMD|16jJ5O);8kzF#>6 zqFfb?DZ%NlbPiRH)Ex7`@8=HJC-CJY#oZd{D)Ww~3pCln6X&#XXa>F^wnCjtPQ>vt z5H$$!Ij3t)yL@DBtjwh_p+%j{XE?|ItppG-<_`FIQ`K(VZrh%jicZAyPs$`9crTW%do<}GY%h0>yzPH;^!F6yC(nwV zk`!pMI%_mlNegroE<2$)l+QiZWd0rBcUl4d#)b_=3%Fzx*5h&E^(6J|*Cu>)Yn!Hj z6sK$bOR?T_8hbqZ>v?IsF|$7NKJOs8qv^wfg|48$TeS%0Ua%FTPo(d|nBt!56-}U3 z@aFcgb|bG&QK?u`AgvR10(+vw&ajg&y42^eecb)C3I5xFpHUU`y+dmRxmcP7uoAoL zW^$(DRqP{|5FgeBnx~`jM9M>J!|ZP1Vrq)xsB`^41<@!gu97xPi;>a$s?M|HLpQ6C zlvTFLkVvCEOfKV%{IvT#Z1oDYv#M^FyLO8yEZS8S8kKUiPaJ2LI?u$cNdfC1m6&&3 z^?%`sq{Nxpc_tgk&Clb0$eQjgf%le#8kRTdL3z5&(a5PO%Ln2oLl)cDq4oS^O7hpU zPinc@d=6$;@R}e-+uH>Er@XzeNvmeeDakVCxs2H4T#N;E?P@u@-S<^&WeW;o1b-S8 z9>2|=UfS@3)A5RlSH>oV&!q~)4}~?~1K)=1Dg&rN%}2IJXZ3#alL+y;#lSlN>oq2U zlXX%5d&lEncEPmK!y4vCEbUWO!NNt3xhcV#MYQl$27Ec(=Jm93G;ZF<%EdG)fkgpFX5ZbJ zuS*&iwoX=8aLWTTOo}vdd4j#lwfU=Xh}Aqh>K2aKM;r9)*3Z8w5lhda7uTJ(hnS~W zO*DA3&SqRqDr6L_9kbhQ;9Ph@SnV8mBJ2PCL303YjMaiY!O1HK^^x=0MhFwh_r3jkJ5vZqv z*#U;ORwg{tbRlIOhIraUB?XQrB z#ESBmvr)A=i9`tp;{~Y2em{vp%u4QEmsx^pLs3nw2+stCOpysdOiYelg zVGfrJll}9?>n-3K_d1t2Ti7a2oXQ#3PcoPFef8d1#p6zvFM{)*;x~tf2gNG}zraJp zHgW2~?O*vD^JLyUvh(I=We@nTL?02GC0i`Dk&woPN`~`TZ+MnE9QeR=OI^D(F`N$*Q8(iDq=AP5?#G_an z*b%2_uX?H~zu_MJ#&uP^iYGDdD8f}bqhN9>-J@uwW#4mG`JomU7``sCk#LF8G-Kn! z(5$iH%^vOLe_`xn#?d?ieDQxz^W?JIFM2WEq-Y*S-RmbY3N1!q2qjtlaicLq=TShG zFrMh{m`oeZa^R>*cXj~ht{|soTB>!SR)n&O=>seC8?J17ND&TqM{^mf>n}9tf5AIg z?GScCxaq-NLOks-8aW93(iZ+dV1>{wgOwD2%;kJ12qNq@uEt{T!zd2K_x*(Vz`@yR zvi~mMr+MuAv?FpXWl{5A0MSp89h&H0#s+w3Jtof>q}y%Re1cy!V%0y?yK%aojoEy8 zjfP2rc#VohtO@Y~$T{JMa-bB)ma&`vOWJ1Ef!N193)G&@>L!7w%qDQy#~fhW&co51h4Nl%gislmH@F=&%D z8Ln>Ut@=NVUS+{9_o`ki-_P4={mr)LKrBO0V#4dL%zcr3s-^UX2U$l7Rr7;# zMJc=0g0L=|PEqMekDlSd?SV&*P!{TBCUsJ2+qgn&Ab7ox(CQ)R)3Zg@d9-Z8(NBQu z?cOkNyV-K30J z?37GmBGEv!+s- z#`IIEsETO1iq8?yY5mQ7HZ-9g#^+0@xs0{m)N~1SR`U%v1#Vo^IsLX$3u*?R<7mOW zl~if;8$tUssK0?R%z>uxmZ`s}>zei#@a{a`oz*mr@vNc84BA}BTM-_^GFncfWomx` z)XZo`_h5R7g05oZE1>4{yzb5P9s^wkO<^Pnj3fqX%HtSw0_`WjO`x3wmv3TyRzQ;& z`5fpHXdHAE)U=<%TBT4Yjn>k7PG*jqn9&Sq40INB4s-!D4!Q!G0bK{B{o2H8&}vVE z#z1F5=RoH{7eE(5BT?c&_ zbQ62D2)YEC09^sy#IDf}&STdWKo>!mAZZa$lDZh^Ea)8QJm>;w0(2QP33>^16*L99 z2D%Qq0SQ?KC22{6u0c)`pevx4ATu%070ApS=sYON(lRJX_&VqY4H^ZV2VDSN z0woDdfv$mOAaxPYC}<3H7IY4D36xG^67&)%o$eIq8Ys244(W}6MnPksv!HXJ3!rh( zRZ!Ey3g(!^yGgv2fTloAofNn!j3I@cO`&EAZKg1W6l5TU zIi@h46#7b`uN2x$VT36@aT(BuK{r5+bfwXLn)M8w$9d32&?V3~=rZUEXa>}@u!gtR z&}IfB&tT*kj5z~t24l`(t=7@k2DlrLs7<`JiSch@1@6}WUR36Mq*241|1|b!0Z*e* z(r7V-=ell<>e=TiKZ!rh<VaD!wBdM(lf$^99f809B@C%%!1C_k5Y4{3Z&wWS>N=;~(`?>P8uR$)rlTf-`rR1Msy zE{o~6TzoF-^0F%E&-Y-3?^ij@YFizsJ=MU>99^fO|IBP0T}r*gb;*{l-EBQ5deV$tU9YLXIanWq zQ|{7#j-F=?EhoV#VD=^bULnxihdupK*%Zp`F5IO*<@K9YPuaAlyBJx!YzC52@|4YK zdL3hFm(645c~9Aj9&=r{=q+=3X8ti>6w2-q%042L-7A!RR4BVoDEpXD_Hm)?eql|6 zW1bSmJSB{IN*ME$Fy<*?%u~Xcr-U(2Vay~!5y-CD4X0IR&iFpgdC4am75%T{la7eG zqFZ--@v;m#iS(?&P zUkkLG)@8f;oYw-SR)T}l8a!jBRFdM0qD)9FOo=T_g|;Bc)kR2_-F2g%uS37I_dSY} z-)NmgDvd!tj7D{_^HiF|+LqL1XmhhVm(gYpE40|ML*DW@R<7WapE9JX0e#w8PIlwK zw^Jl-A2FZ1QPo1f;JD zE$R5moBDGOGwv*3)z1UWxagBi)An|t-!#hi0^?8Xl48sHcb1zIS;6V)EMLPqRP?HP z+u1-1w%=*AcpdXGYn@~J=qY zBLfCm?6cPNT7i<@XCo(dMjd`_k%q*TSzk2K8zB^H_oM8~d-Z7@+DFo;#4CurY3!2& z$&`5J+%yIKO}w3!zVY&m^wpI`CSFZlgKU(AvCQ-MU7mXy^W*NS;mw>ypEY`#qaFe4c6m#V$a;iQ~6j#@Zb4 zKDa7qZyM~K<$W4@iTg5GdRzI?D9HlJr`H}`f}U%z*7l%G`+zuMT{kAAEwclou8LS z-l@!6WPL_5@8!iIn_HsYi=#9z@)EE}Mq2X%kNh|rRrB-e;Oq;Q(FJ;Fa;(T}8=VgQ z>F4d@^N#Rd8n)YM#h<)ZwS<+f3T>{VFC&A&ac*LiF8r?`ugPb)DA?+vYNOqt9j_p& zRue326gEe;zA^pDfyI-oxr6aE1zwW1tHkS!SEU7`yf+T8}qnv)*;{XZDRx1Hbxz&;Zzdf+t%$8ue()hdw99#<3`zCTB zX?LUf&2C#=?Bx>r%)!f%xd$E5EkC!3b(byG45YT+LeqY(k$J*6LGB!^k;7-g&ov0( zYADx8bQ8K#^2`=Fx?xd=-#c@(rZsYSUEosW%tRJyAzbpUBwsPX-}2&^ms_?*8dZ{8 zmi@+6{f$nIwEKp;jZ-1Ln}O7A2W(Y@tx=J8*Xw&4JwP$|D=O%Fk(MA2G{~Lhd2F*S z3dY6F8(DaL{&yD{O%W!xIH%LrSD^760kIGgD28BxBzm*)49DlOB@yeaBH!Z#Ms-P- zUe|x+R&K_R2*_gWDw_>ybFJ>TJ<|?0ICIiaHag-!H-usRb{xs;OZp?1Uamia>E-$( znSOjg7TFaoA?c&b%0|7sT)zh%w6HGrO7e8mmQC<7SeCeU*;#9k+$ zZ}t_T)Sy+aA;MWul5d`5Tem0nOVL-GK00}z@btS{MW$`0lfp>v5(7N!=0xT4s>I(8roui5cQ?ayHMT|^2RA}Xut-W!Yx zB)-dgBcq&owpqq9ToaMo71U^k^luSX*!m|F#WV@+${397*7cD0Yrw)`>$eP@*O_$+ zM)2u$6vg&ea6)MY59#CvDcvKHkWd8KS(0*zZ!wJZ&m<&OI2238 zkW;Ypm*OR~M#T_*677fcQ>>qh0mS7Wt)CyvqDiQ08@=U)IKwn!=Q`?1ah)c5+EWD) zw@u@1Li7cpf6`ucmhfhPKZl5l3mYKCfaZBz=KWpRFE2kB1C)`YiN9PGVm>woQ)f@w z&tKEBRm9nLq30xoSjP9OEPuf)CPfWR{zSrKbFf~`7RlgnR8^b+hKW* zNC>{LS%?khdv7vKdFD&6;Du;|C!J~A6_N9ChVd|`wjT(GaIg(Jv&Ntad3g!x& zmc+T4WD0mJ`R3((h++6SAnP;?7ZxC7_7LAvNOA^>QwS&|p>3U6Sq4 z(-2ioOP|R#OdRa0vNs5%8Yy_5`zja<$+A|>awgYnHJ4bQ8OwU0C{pZ=Tr2}kldru6 zS@hQ+%S&3-QPLf-3~G?xt1-IV6|E~}1}bbvq#6mfO{JE0@YhJOjxV>?G-yK!$GXs? za*Y|xVqe*~RWix{@2#PrwiO)e&8hrXX>)2b}HAH%9jO8Sh zKy6UAPO?W4Wixq>3X6c7?1+i^XIQSn{Q8EnaaZYYUW3&+tWHxojQyMt97T1^sl38?OGY0wchYuYxs7TIEfvJQun438%`M-1 zuo0Z3q8n;5K4>7uQ$&AqoF&6@TtgnnW|$<)Yi8G(qmt@ShO%;KI~kv4?7X2yt50a7 z!nYJ88*M!6>sUW~XRQ6p6PtUqzhS7d6iGfF#lY+6)&9D|>!;DDD(8VEU(Ss68f1s! zi7pUO(BG;8nUMXc3O<*~$5xP;qB6pj)Z zZ%Yw#m(g;acS$&p8)cznOsyU}zwvyS_ zuad%aQJ#I6W{9^m&j^zcZ8=0oEHNS~@G9?!J;u?GSy%6RNijt_ch<|2H6S5bx~D4Z z+qMogJ({`7d1R_pXI*JyZNZ$d+py)8q>T}xkEH>1}l+M z?R!dozlkzG1-8xh#+j^rbGY|UG3~Hgt2kvv)xP1>&aImH5JumHZaGlF-%zaq0aI2v zsq76ve-?KGZ8&V9N*rZ!xt8BcS?th#2g?33%wKC2c5WBwt9>6q@_4cw{c2}6q#K=d zIvkcAxz;oz6BkqUn#$K=;i z==&;bSm|y{TWc;Wig?N~n>e&wmg8Y09bxFpdM_qkVCEyCt0YT(6|-*enK)c$WYBLh zM7K~pmujx^2qf)7==*?ch28lv_Z+u5B13n0_e3q4=KCdt2Eq~W2F6}SWMvn#wl`$j z`o-QZ+{P)Uh+z16jw*_d)4I5aoWi)GOkEaU-hODk1jcAF7M)jb)Su#VmsO4D+&&rJ zdMRapuh@WJRDjnn4y^BM6=&NGyoj2FAoK=-G!d@KLzJYL~ zn)WWAG^1oI%aetyl1-3m(l1u%mQ@x#lz~GIi_B)BLmlXT$=^%i6uYeJ4mi@DKh%TD z0Tlcg<`k+840@)ICY3M`5-g6(LELEog3ZFklslHEi^mO>5bo6ih%tS(1`!)I%A%%tJu3~`t~F{dQY{sZYNz9o3% z`ps`1dH2X$M}BzZ#{=1(`%bIq}wscTT+D_t5G6uHPAWa;U#=d1$<6=eGB{zS8%|SbyJ6-?hHS`@S&n z+anW0cMLo~^g!QpV-qL8H+J_>eCUyZKkL~aD))SPYtv1F!eJIeGDH zd2)8(Ux%I?yEgRf!2b-rF!sdRPYu0%@&`k&4IUkOYw{NdhbH$1Cx_k{sgHbmN8-8H$bHnk8t7AVJ z`P1P?#(M^znfUDKS4LhPE|1?f@$ASO6JHzr%E-SDKi2cr;U_OVcH!yU2Zx^>eChN* zPJC%`n-=6sH>Hj)&)8yCAj7)xe@Rh;8ANk3|+ruwUzA@TAdh+a><9AH{=jfe% zKRWyF=!2ulQD^)YCVq0R>&%C5{??fXhhH0hYxte<{^ZybUEe(RRM+43A07N**Bd=A_5AhG$9ulh z^G{vh?|G-|2mL3G{oXD8$8S1*qUV2)eYOAev2Psx{IPHL-+BDZ(Pz5;ue*2j8^`ZH z`un|I{SWjG^)DWO@Yrj|mwQk5P91-=|Ka}2y`LC;W&G*hx$awzy?*SUd(-`m?lav} zz0UaaqyKR9#iK9XGJd>#{FBH2qyJOKpXjZ2-`^eU|NQaaI`-rKFAXG5&2?Yuf1&r2 z-H-PFd3UMzGhH`z4V>B>{kL20=x+AEdi)PZ-|Ky<`&Um@dY}7$_PzryYNTy^l3m(T z3@A&N(5n)VCLjc9Vq+0hET?NZ(K}CLdDXMVuBWG~c=i&+iiN1CAeOZvin?}e7(MSS zr(W!cDBmQzNYgt{@BjVZd%rjHd-lmZnan)%l$mEH*sO)fb;b(-sxWpc-ItmSMMYs=*}LvTIZ8aw9F z1g($&gCA`s87whaE0IbJB_^=bQ_vF2&DcfPN7x<9y-vHaXHF-La_slwJ`$y*k;GbJ zFTpJqNM=c;k|)ys(nI<-`mXwZ(qic)DJOj(eJ*{cXQ*c(9ja%qAEqCtuh8$N-%rn7 zkI)O#i`8qdm!{WSZ=lp)woYm!c`eH|*d`6L$uLK4)|fmsH@7S{1@@;cuiNajd}vuA zH?ikT&1{az?d86Bq&&jZ9#1ydXSCPU+vc)qsQpvZc%veldq$T`Q%!rD;|4`WKij-E zr3@|_+%g^SAeFZ@_j8b&&Nkw0W#*BlOXb~d?c@WE-kP^F-C*9`*4K2G=@AD{({oaD z&{T>@UPuQ@M@WNY*%DW2jKovoE46XDY0BBQkjYFR+qO4&Bp++r)BIcW(UPwv`z3@$ zU&$idQppNQAj(NXE!Nml<}=L~OClsBaFpDa6r*<}v65Djwi1O!uFT4IqDxDQfu&;{iVe;9AUbgcc+RB$ZJ(IcGX4&TAea&~-UNS%G(8K(KnYGMczS;a|2g=Od_NaWX z`BRt94wEbpvp@^X!qH)YEK=6SqN%}a$$4p8Nvb4G(%r11EKOc4``TixY^cR-vzD?6 z7RzO`EjG)R$gfMTN=@}f80zT{(H|>+s6Sngc35X|Zqg_x1pil>706G`^0?$q()4^t>6rF|+N3MZI=uFfKMZsId2)z$S`W5I( z)Eu^kfScetz+l@SOTVGfzyOxRfE>I67~%?Z7}6Y48VYAXj_v{tk)g;jNRdNILnJ~h z(GmngEZm78$pQr(Cteq7G7zpw-XQ~#Dd-I_7Fh@>Bj8n}UJ`0p09(@K5o^Q}4G_*y z8rF!zc6g)`-YFN$2sOyjP_g6=d(j>6Y>C(}*b_7#!WCT~4SIBGHaHF^{{ zBin!sIKy-TCTC<5A^}g(`9OvgBe{r$MHhHA1lR+6qy&)YWC+W^CG;maS004agK%b? z0k6wIH}N-;+JY<0db8jTg(&q9Uy zS_U*=DZIZBo}B_3WH6WmYZZR^DoMB>wk2>QLyE*2vLK{`Fr)#rz9=vWrW_Ck<{>|# zJ&}H*<$ij?04YjcNSo0Mg6!a@y(q=n$p@wpU=f&#UJw%Wjak~WzahDR^RRb6fTn0W z@IBfel%uVY)i8EskoI5}G6wkH^A-@o(B7aaatkC%UV_Jn6Y>DkVxSbJ zB9D+($Q|?+kfDRXbD=cy5&>v;q${`$7%_hlWhIZ%U*LHGvK;0K*q5fT643qxEdCa;F$5zBNF_V~ zu*ZS;`^`X7pJs;ro0dl*`Bt6sM)hgcxF#I3I`xAZ<%P1hwb$G?*XO5-R$X4l z6TB8m>e`c3)4T~S?L+*C2U44#D%ywoE7H)~{jH*Xc6r$c^?#tQdiM4q{y$Q_o;+*V z+h4N(Dp}O5|Fy0h_^^E5AA(NgAL;KO)nERB@{&6I|EYYyU$mc3vuE+XFZA~+-PDx7 z=+kxix6;buPx|WsR3HN^1WUkjuo}h;+rVzH7aRb`z)5fxTmm-$2kwF2VVqZvND(8X z0b+$XATEeI;>7{^5D4QS%!Ke0giHuqAml?h3gIk-8xZb8c)iVar!=?}{1v`OY2f-bJ@GFqQZ$X7X@pKbp zH2>H6!f!VtNVYh?NGmHVLP5<#3R(jKv;@J~*|3e*9I)2IfVBa{f?fKpB>LrM1&*P! z&Q~bbTIPVYJN-24Uf!nHFYKe(P$CB#M=sQ4?^y;mb+yxMUUr(^5}u^kIt~Rn_LDSh zk06?TyGya1+(U2Y!WFqrk3sIiL7H6>CwkZF)r#GBI)c2(7d83Eu7W-Nx@rnGn9_R_ z<|y{fj0XEX4{8p4Gn`f%y-*x%`iMCmNgz0-9{yA@l=5|56g}(TCRfV%~L9pmL-hhmtFRymhYWMtx&h(SI&G#tzx`+M!A;Hq)qtMO)dDW(QWxPK@<5k zdHU3`N`0y@Z4X^|)rdJWKM>RW0KnOya_n4tSL#9=1L~L4=fSh?ERT*l$V+05^QLDm z1G6o$e1lP40oK0@W-%g$YS^zO-KghS%rdDb(D`HKx5pE?8vS>c)ajs94<=~T@LV{7 z1hJe6y#0FpJ`28u`q%I?9flxV5YH2Mng8o@l~H;xEW5@b&=E9S zC*vo=voZiNU#rP#8Lx%)ZGw1<&hG`VfF>=}KmK?|E_b|qjHamKckV>tJsb-{MXmd#*V%w;!k3_`T-r6eV}=hb@YG!F`nXo;OiEFwR##xTw_p_OP6Pxbr*k z=*$HE@uH{TcXkKGfA2;=nfELIv}+geY?v+eJbX6&qT?#+Wn**tRq#3fRb?D_ZLLqe zzI6hWzi!UIIrNsUxVE2vyLJ-&Ztqj7a@sBcR=V*>`h6Om(1nsDIsj>}QcBNfHPDYS z=M79Q(uS^al;L9|Ak!Pg%g#j5#&_THCfg3urbYgg+58uvL3Upr8}3P)&st1bbnFfq z4sxIxh1>#`NeW(W(}=cWGI*=3Z>@OJwFZGVxX999K_#=Ek3M>?Ix z7hR^DdMyLaqq|csu{dz;?LoP@rO@sXhba&J>A=(BCGUB65A6-c@ZKk)XrEg-l<%ft zpvj@zeAC(MXg{VQM|^9d@t^R53uygV)2!!ZF3?+}35v?+f@Rw^A)c3%A^XbJ5+0X<82H&c#G@)Wjx7a&Z=K z74iN!7tej8Xl0Iat#+Gh68Sq^>mTzKZLT89w!QXg+RjYl+O8>Ar2H$COW8bAlX7*h zGSzjGqRP(4?Q6FQo=^CT{{43+(Ut*l=%_W`syRl$&`0#dF;=0oXE;At@8f_?1O{xP zAXdVHJr*XvUVf~y5nY(JnJT>Uh(4DYz+Vem1Fk*x;NQ3nCEpzSoihA(3T=4oJKp%7 z2{7rEz?&9D(Pnd=QVq^0)0jsVWxIPFXly;0Zx#J3-KxNuYVBeRTK{;FYI9vpxBcoD zs@w|ExRM#m^o}};_a5nr_mvN1`o_+~zkbvR z?`Idvr07OL|E^CjX0Y)cW^kAhPF+c6hF~l+ zH0L!wwA6taK8<09=N`pJq?$4#W({OUdGEwWrCnjZFLTC6`%T2hoGxOV z{ET^9#r%vSQxX5rzmrJ+Vc}>oKRfcR7Jug~(&w0bBj)GC6p485&lVzGWgaZ%=jG>X z@!e99o-y5B%xCP^^7+$Or$zctv!06iMLywTeo=2NpG%P2B7F&AAm*2r{2=0G`dU7h zuYD!vm*3FxxpM4Ek-jqfu$X63;m|t6= z<#YYA!6JQqQHGe`I8?61jF!($QIo~|rV&~`x5#}&`j*%vF~9ZUIT7dBYWZaMM2UI! zUSBc4U3FKB4;YK|+#ac7KKCarpSuFzi1b~3a50}}oTbIhwS4AZ!NmL?Ov`6M&Tf%j zaO1j|-#4wN7UydDJdjc><_}Ed#r#373pW^ zeJAG69@p}DzJDW;etuzdF@G^~uNDu~@_E_BOUz%6(ehbxTO-o1U@ye{)%;Ofd{@io z^=ZF|`RhB4g#3|uBVgSC;~$WKxV9*}1L_^R*hV;n_@-M`d}A{h!f6cHF5`ghDR|Xq z<)wd7t1<^Ev-N(bvhNN8TZ-26Th|AG943ikr#AxIW{w8iDT4k>WB6R_O0H!T0C^?_V@Feoj-)B7_d*_fc<2kU1K+f z;0vJ%1ieY|XZja!olT!wIhZ;y&U~V;m7kUu0BlKZyO!2k z{735}`wR$+AgqA=WI?~toPi^* zFvMGOMj%Ba1FoFx)^vq2r_Y%jF;SQvxW}2TjnXvOsHeoH4O5uUxT~~CU#DrDZo^GBt%$$X8C;b5iBqhku&C$l&^O&f*%+3z@qZR0hebR1r_pYo^C zg`}$Q4r@Nx=c>KT_xl|7yJ3H?LHrA(y1@Q|e~12dDZ3-AwO#(azlg5Cjpxc&L?<%?T|WdFT0sze zOyH#{5dJj}Ado*4`ZEg4p+b4-w#Qd1at|%l6wXU1dE|HNf#uhI#zFW2@-r7=VLwhX|5v)NMZY0bL-$f;js6L%1Kl2pnSCw&RK) zt%rD{w!S%Z*TflEckcof(2X}eBHj{ptYwj1nDC(Du#oTZ0kMZ*PEtOV= zuPCc>AI|!LH*a&Tr_#1Khqv2zM`@osns-=b#Wl__rW_|dR^n7^%Bk0Q&bh-G$|Y99 zxdx)Vn_Gy|o!rd_w11*(78^_j27SXRRxAO@saNO@19Isa{lnjTS$)4lt9576{e!ao zux*}&Z~^xH6(+o{~iEd%)CyKS&j8?98Q4#~l(Vir5S zItYk+)iZzQw0qTZrD@?!-fZS(O1p1qOF3tS@h%abysNJb?`ALK#r;M-531#GQ$8kw z2C;5;RdF$Iv3R{ARf5Y+EaCC5Sfb?M`!!0J_=Yw8&H2yg*248{BCx>EbTomZCDHZGTxStZdpQGE1sv*xH ziFVijFC}Sq)iVOj@WXcYk@>y0%#$HZgEF55@lv3THNt`L-4=ifZx}01&WzQZ>!l#y zWP^<*3T#OMX+Cq0!s7cCWWxc=6^*(~AT1NeE9A|4kyc(x zg|+iIwT)3rg)Q1sZTnQAw!7VowLcf2b~t3oHs0Py?U?zB#YZHNP7|-N&YiqSmv3GY zuEEBnTk8_S-SQFX;k}3Os5DZ0$(9pdrN61YAB-n__McMwo?pr~S(&45x??QsH-4U) z$mqfPr*9<#hLLQu1Qi*W;>-ql3?+k`N!bvCR5H}wi3q(LOg68O5MhPZWcamThzK@N z-J+n7h*ZhdQOhQH{;8%F1C%ZC=38HVk$91FfrV zWCFYfFCr8$D1kPz6~cB7I7%?!XbiEF&Sr%DhX`mh=~|l!R-fzDl6aTbhpgNXPk_a7 zBr?O3Ku39yk|D9Iv{#H;FWH^dk9JcV_#3hYKlW4`VGmfNE!RjHU(6bFC)Flb?h~f! zEo!s;lSG4!CrE6`7Q%eyX42yOxkSSObI3+rh7gvC6e(|>LRfiqAg!GTvo=P7q%E4v z+CFW`+TC_z?9aKf4u=ew#@qE-$ISZ-KB76{H1QZ~Ah6orIz%)yk zL=u#GL|o_9v6<*rHSOh3xA#8VUXGMi+e5Wo zKs-vDpGZ~XkJ-yas@h&0!y&-<&QS&J1+F6;h3AIN7G@78@){&^d2dN3|4AURr*t@e zHFPU+^|x~3y3H;2dVUxD#v4nbbXh5W^TtwwOKpZfUsTDy(CdM}RQDxbnKfq0l6{G? z!lQV(>>OLZp^SO+P@kxnHI8v8nnN@mb(wKwf>^xS8OCYi0K(b$IOCFru`ahM+%4oO z>vo|KC+P19e}{HVKxdq7rvD8d7@A51mL0@{aYrKfOn)ZiHJKH ziAVs%e|C;oJ%28-YfJ}nw`~!d*LW_OcYz}ISdCNf*_p)_lx-vTF1$?ayK+apf0T?p zP+(1}yUt|~u2?}H3O`I79D~P=i(^vhxg(2~L)l2mYy@ABb0E~S3>ngUaVT`)0@I3MQ`AD+-d@%dw zaFDuUZwm2ta|iO>@*tveb|7hx)tWVQ9n82a*vYzL3m8!c{0PRwsR7}kNn^ag6T*uP zW4yWZgwK-BxG4Jr-fY51BR1js1iV%GYBq6iFT8609J+i%+Zz7R;?wsBY8kymXZoV; zS5_7UYISf}r;9>$H5PoJj*e=rlW|K3HV`^t0Eg=fe4tKWLL4LfcY4qNEPYjyAJP|D z@Pr^Q8mp`HR;{lb-!Ib_s?`@-Q(uQtzd&CTAxwpE2Nv$;y{s_Vo3^@65z$r>M*2O}Y3nASS>@QqVD z8Bce2@?-PQ$Hz0a=6DwQ8RMC>=76Tv(ZRnqp4D2%Ipslk3H1YG1!ohT@5DpM{EryN zYywG-Q-@j4@^rNq+luw(ZjrV0#5QE3J4Jp#Pur!RGvA1(_2!!cV?W=AoN)*)5R@2j z?gR1g{|LSON&NP4|JmO6Z;9V(>&O1{{l|GGgkw-QCn3K6A2Hv2;@n?jzPa$Z^UeGC zta`rL`~|w;IpE@h0T&ntxJ3P*nQQi}VxuJlSzj;P_SV@i`kUsP14lkzM=mK4=0P37 zHNDG*f4q*~uC1P96719;)JaAKlW@KR+v>>(GI2*B+xl!8Sz~>Z`z87jPDI^^r1$Ya z@fYZ4FNAWapGpq63j4y5|7iVu>e}<(A;j?)vBJ$JG@=9}Kvr zLpYSd4i;17S-nu{`CKIpDtK0QX1LmT z(FxXM+iSJysCR@}Mia6@SAP~8u2h@%jbSZ1E+8AmA7>kd+K`qW6@=WTt=h`SkF_eN zN$UrF2%Bp=)V9U<2)q5EYWuBrti!5-WaGI>gd@F5jZ>3Yr(Wkt=M)3hB{q$84GbsT zqQgmd%R#J%{sGeCK&Qfejw37!I3retN6R$PX*#THbwj&=^5z1fA>+hPhF^1 z0?9WNFersOevSdR1P-`$*R7qBAPC=uk*>_kq*v{EO&u@mtUTAth<|UlkEit2^+!+O zq555)Y+W3>iK)f{ANj1vN3C^%+h7PoA&i9WkAZl+VE+rVzqSAWy+1Dc^S+<8{=8ro z?)|`&_4zh{@m236n)Ew@H)R$Pe(~$Rcpts}(pGFz;(WZe|9?Tsu6lRl+MR0u{?OjK z+gvDf=-+OFe|xRnb&twOv=$48vOA(WBes zb;R19YRI=+>;>9ye!=VH{hyiyi4!8^N1sUZ{?emKnc*>bAn5}8B2yoc&K^pTBw>gVcg{EQ8N@XJ=vhMpq zM)h@d-0nlZTdgbiFxV#aPxm(Pvctmc;luy8{i6o|4xeU!=Vt%0{keCBeHsa|pp%e{ z9#_~^tHL=ZF1aLgf&;g@>7cZ%9+li0Gez3k$Xndn#{p^UJ>DtTpKg@4(STEK-10Il z`|cs-rbQMdn+w+{w~TmRvUUA5WlpC{CG7P6%5A}UCEKYf+z$EDl3YbUF8Afwl3heA zZr9BvX?d78mv>-HT0U>W?OF9zTEP`B<=zQVY5VexmHT@-r5#xONU4sJq#Yb}MtR87 zvE*>tHsujRq~vI5hVtm$pGyjza=606qa{ViJg#W{`jQj3hH)oVN79N9cH~Y`>(UH* zUsD?7YZTVKhA6E!h1Au5$Y*%9r0Siv%E~Q@`s1sC;d|SymhWsR-$IBlX~*%Y%2j)A zC{E6}tvsiEq^TOmd$ee7Qingi+Il=O{DZbxD_(HF1K~bw?{|ow!eUMfvnOzQ4Lb9A zZ`-r^Pg;_DN+%IlL-&$b9~zR^ZGKl@&rc_Aym28*mpvqI-q=8LsnNvq#irVLOg)r* zW#-G4C5MqhJXS8_)a4uX**6axkrlIMu?|Jc$;P98VI7$mHQww7>ojpB>1=#SJFn+u z5Y2BsA)DWKV0Ha*`t&ig5dUL8^Z*#}aDiBO4^^@_ts~_v8K&^r|5eTT{r&r8m6dyR z#}lPD`r)JrHYy6&15cprP>hR7C$-DM0Kv0H7GCp7xv~} z8mK7gjJ-V`2Hxd0$0|3#sdlj)hD?74(2;L3$q-vw+RH|zr>LOy$u}wkqLen+?x-@t z)U?r-ZJ3N-O&fEWDw8V*fT?=2$}B$s*pkV>eC8y~;`^^b!vTG9<=RKca`03BijC)p~~UlZ*=2r4^)nsr)YeH1Lia_hj#8* ziMf1J3|xb5VQ#Ir0(Z;9n1{Ctc)Yu*@{$b&UZn?B-VaiN&;Bfx?|F)DvT~}b>5dMx z-}rPDkrDXmbIh^FSopPqPo87qo*l&!nuuGNTDzCps&)G6iqw3T^S36kepzK@MuXb( zkAQ`Fr}jKl_xmXmAYJ+XT>_7G5Y|I`*#z-ncw1!V(^GVneZ2DPN|kv|<^S#V_v<^* z{LKkIOnP7OxjL-BAN1)FwNy`ETG#nu9cDmx)Q=4WY>$65o~;)v2s)E&)amRB20ZOJ z;3>Sj-fdy_=)nK&G0`QhvD)SKc)~3dsNF7HBs`oPNRNF_SaBa=!4kszft}jtTaxuv zA10ghJ4ZBS){=hlxdcH^C;c6|vEmv^e-sfI+M5*DPnFgCo%R2?zc zo^5et6&cz79UC>@ge04CZ1i+}vZd8qLRdq^yt+xmwdog&Ui?4D;=cn-v2aF!#=uwQ9S)8GlzDZFE0M$dBZaf+Cdh?1@N-> z+Fe(b@@W^9{^{v_z|eAKvxG!Gu)}p_kcST+bOb1a1B$5-1A7o^caMMXFE6BY_^v~I zL{0_WqF@dmIk%LKT1N5YU^N{*Ri0;m~NCyg=&irJU>M>@41gk*ypKXFKG4q@2Bgmsda1V z_k4SK!@B$+_6OScCwjx-dK|3c8+g8TSjAtTJ|DUvp2?^AuP1-d^ybrT!^rg78AOK07&0SQO59r4 zND483_L6GoKKh)05cB}bLGJJmozFqJPy(N)y$8zUw(yS{&ye}dBEFz-7x_4zA_`NN zlTV@+5?t&=vM6*sQ5-p(Eb$paJpIC${P8n)zEtWCJsUcZFB1)fe$s{bauouYC04P`#OEY}Bmn{LfFrdUHwaT+|dTLV&9`-)c#e+4Oh*_-GwRh{gK zy(E;;7o>`WB~&(v$X@U%qSrqTL2CSVLY>=2_RdS>HJBx2pYvOJ&G@Ylyk{}rH)=7| zZ}|k^V<$lULtJ=m9~Vexm;tW~L`lzND50+^gA5e)34g*FFP@tXA@(#!S*u?QgtgVKQk|?!{ZD z?jUWlrxCWX;iO%%Cn1guAtl?)3Hw<~A%{h(grhVVavEDsIE!2%xi%w>gWU2@e6Y>% zsy+uvZRfS;c)n0-6!bUY10$4=XJvo8O%~#$`|q$9M~o+ zAvWm$OL>xa*W|-;Bf^-?(Ot8R?f4Kac68rH`IzL_@)gY(M95cUgZmx=Lsou(2ZBp_ z9V*lD?>oc??|W`Orln(zYwf}?(Jl_Rn17bF48Mw7Ud>~>;VU`$iX=yQbK7ATt!9Gv zTm*QyAWJbhenf1YTL<{$Ii;gjK(TL zDZP>eNpc^bi9@XUd?;i8d_OrHrz4=mA&isDVkfwDEy_3R!B6 z3|e`KGMS)(n9g|wn>oKi2K!!w&9yTTkyR`#YJ5Rks*$i|;XT@_d>U+>nnc@VM^d)2 z+i1IFoDxSap(Wc!Qued9A`XkpC`ajH#A%!|v;n(`skK*>3Xi9g$prf*L=ad`GJIeV{E2h^lkC0 z-A-*!ZQE|AnA)A%wrx#q+qP}n?pwe0Tim(HO>S=TPwtnM^_=tJK@=KI2lv^T7#O zO&0@k;TeC*mlBWV<-@DG&np#js_NQs>-~`ZHIhi`5;>=Nd4X;B7@w-BR>6+rxyppf zh|xi2`a8~{BMMC=w**Ou$yS^;$!Z_llHkEfj_89{L=tGAA3#@xa8Ze=QGN-9_zCo>#h`_l&i`4>K;*rgvgt1pTD{g?oG5!inozq0k?{Gjd}u+6@5J#x z!Y$+#OK8=nN;!Z3UHfF<_k*7k(60xh>jJnOe>%5&Z@p)m7JhbYvM}2rgpVSe8xRM^ z10r%L5HwgX%R$iuvYn_Kux7N6`SQ6~6Ds{VK-v~N;}*o`?dRZD!N;H6=GLmE`{wio zD~k~%Rin^wOi_3jXL@C>g(f6he&uNAZOc5*aZgw>G(+>9tvh5t>N`0QVC(2rW_$IU z>fx#RMYAU~9JTb^aO@iPbn3#e;wgY3r)%&8WuIRjhp$$F7AC>uP&5s1+@Iq8+F#ri zavw-Rpap*)tZp}?5ESLP!~>U|0d)1Nzr~u>1^r}#KviD$h^<#c_@`=)M^!%&;{hzU zgeW|FZ2v3Lua_eViYEVro!}Jc=tv?(u6X^0iL^U%hjym&9!0$Bkm#&1b-eRn!t)Z6 zHhhOb@gl=6=P@jQG!Io4fmWryPY&6NQBBs^|EEr zKvM?oJ~F-IL`s|UCx4?gPwA@p%QFDpa9n*_q&2t38s#b~$U#u>kKTbEBPV6SNbl9I z=z><&L@;1V`)Vp1@iGF)@o?U5e#df6{wMoLfxGsecG=9H=)aVH9%=C99lvt2_^j)2 zJeQ33ywCG-o^S8FTV(Cp91kf;-m^^GU~q;1SRp?8t_8|HQS1ma3VASOJ^3Pe3?hxb z>5J#37;)E&;0ddRzZrBK{NUc(2Z-omDx~uG1*090Am{Vk@f+*B{X%<QcRMCde3s4kUo@u@JHawZCP zU*T=8qKF`Rzgj-IC_~-)vScThV%@TL<1}oe4aXHDgReq*hXharxT8qC5!ywPjhU0{ zRCrWh?CTW)E4D(w|8{?)bdA-zg&Eoed|rnTjb?o<;?VwzRUul4#=7gKeaOCn+ACSk z=WE8xldf=vbGhy2tA}n1y>S);iZ@?DV`wC1kE$j9 z>BjEHPz`q5xB6g}&p{aTKC?CFNJs6Sdn>6n`#?wD#h8AMeUAaOZKtfj6OKNiuo$=a zBJvQXNvf|MGL)GcM_g119w7eMHHJ=M-L5}}GyAI*fzs{-g`YmWKMI3vG@e9TyFmoA zb0&XAU^tUK=M?PQXFAj3?o3vNTU~ySXjegH%7Ea0pnj5C?Rx37O$Ab&=5!;oOS1`# zBo9AIvb9S8tVW$RGJ-n&>3Wnsx;v6B0ohK99J&7Mglcx=juW7wJ&a_{J<(aJceMRL zvnmic6mFn7?uYJ+`4WLUSBQXoM6T`|8H=bH9G@wO4E!-S+QXm2Qg_*tz08?ltCtrb zq&c`{#dCZ+F;N$6^HgB0QQwLWhV8P>9+v%CcILbJZt9kFW^n7jA_(&$-O>0lOTce< z|16k(NWlz)0?q`Hb`_@(x!wMq$AebU@MR<))iafHQg>K;v+C4=m4xdq@X<0AYc%0# zsR}Cn$nPU(B&<>FsPLF1gZ!7yQSRP34cGv;38sj9h`p*# zPoLih_>|OgvtCrY-Te$w_CKoeFQ7uBgNeuHfQ<4vs2>!x{aMkjaE$ORFN|^Qu#EQ$ z4}QE_SRX|r*0(AZ`ZrB}zDrZn0Ou*%s->wE1g9xB;$k5tg_#;@-n1{W*jZmso)ysV z!4gA1p=&+d9-(%tk-eTO>}td>&q{MqT_H+fJ&j^u=&%UiAslE)PW`Th1z++z}eIkG%3N9#%t?A^9McKc&ut z2-DdwlhB~qO6Ik=gve?i+JCmpWb`Z5bo2;)arLxo<@Jg=1V+iWm}d9@G^4a_K6wJ3 zpQ4RVrvJ9V0c&`uO?uI?<|+HLT6&YT2yF&%@&>V2MB>HZv+N$Hch?ov8qd9_0{ zGj_^{-G$C4NovxE?vnQ7Ss@Z9rwHSen1#%EdgDv8p8|zo>mC|#mvUoP^UzZ!HAgf_ z+Xr~s;yW&|Uou%f9dGR%xe7lY&lRV3B_aD|Zp4P(PH%t-=yuJ#A z>5PtSkdwW4l=O|f7rA(S4YM*10y=&Ghc-Tj2cHO*_1T!~z5QER{0)8fidxpvv{jc% zRy(JEjJ%98-&Xs2XB$AQ@?`~I!3-qyz&g0!?e#qSYv|{ryFh4kG$pUJX@vpyyrUu~ z4Viu6?{Zbau-}(yas>EQ=Cq|owf(iC+@^~>9RRP_ zpVyh0vzpB@?ryEy*^=0ZmVIFJWOsrG0SHR&>YF)jJ~^9QWjhjdc)YhvftI#gLp>eU zRk=G4S66eqJN`d#u+O7?fzE~k(k_f)6 zAiRd>HJw}tW(ZPZ6=IzGLPjyUDT%l?LG!A9;>HiB(vQs>P!kz%++ONgGW=X-6q$vUSsuGr)TbX!ox=zMb}ZPet`uqc z6XzwjR~V{QibZJ^Qs`75T;~&nisQ@DnR5<$bthkAC4c!)y%G} zUldjlt6^f%$O7RRBG!CXIPc|J+a2mN<{enzA1HUOs8dDae?kh2s zHJ^f9U*}X$jq4y!_wD&V8u0Edx>w2me$L$|ek;GetP{uh3$0eIr0R~H9A3IRspdGg zeEJE)7>y6a2NR^WUY^V57vY2Dc^hy3W~K4wbF;et!|r=1&ndu%I^zH|V`H2(vCEq2 zx4T6!Wc}xXu4Px3aV5r74+=xc!0V>gS$ag$=GsoKp~D?pKI>{g8e7e8#J|7VYH3;+ z{Sjfyt3D{)6{Qv0!i6((2dG9@j(P5nuFswQ#5K+Lq~+Z$r*-;IwdO5TIe69B>VrTn z2VunPtj35PN2q%4Z6oHiA{ggG0dbS7KKq&rNKjfq!9L!=-Z-{iOAR7CyvJMWHmk-&87mh1M{_~-Ojng8K%!-vZfrAkEgM-cW z56o6c!RaPIE76XH+=@R;Yxxg&M$BL}Q7$wq4|WqzTXIY6)7X_8gULvuu{`=0kG0TL zyzA?9j_cpeMnwzQQ~rH$s`2B5wzGPyjA=8SKM#moWXW2Z(ek;YnWj#s;;j_y%0ZR}&E%gm@PFwfd=i1S& zU-U-bvtP>$eg|rFxx6FePkc@DTGw zqLrJJ-xus_mmH-y_X=Z~+!h+-b@igs-wtwbzc#MVy|5{M#_phNN~ z{3veiZ74Bpk&@S&nLi|Oa3LklbDKV6z=imP>7KJ~@7bR3w>dN1?kU@TVvq&?!Dc6> zuWKIr+5~mW8K>a~bQeSh^r#)xd>%P!UZLrgteF>cEMZlc!=OKvpytkK2Z8#|s%w9l zMT3*Y#KXn@-4pUmcDyJv3?LvI9O#IA4%oNd(J>LcjPcL$6$fb-;0S@dq~e1YR^wIz zOJh?SZ&4UcQOjO_4Od{yCnWF5ZfT16+mGAh-_5-M-pR-4q|s(x#Hfi9gzxB!0*G|> z@J2_FU7uS=*CE^5W_C!I{-9)MVvo;pW1=zRnhP737=ljc&rAB|GE#a{YGREHwCtpFq%6w)LGrvN_WZPc6h#O1N`p z*eskNektPTgbT1+B<^ftYs6nxny3b12&X$z_kovMuui;0=hLQF zJiU8bs=fOY56q?^-1Hldl!|n>?#Zpu459PtLH*#C7o}gQ(j2@khp!I^UqiEwgdgvK zPC#sFi+%>(UI$@9yz;Oo?Y!KNR>V1=KNzclul)|l6RNSUs3c5#1i%U9JtH2wsoJ3_ zxt1xe(-1MV#K|EADm$_TL~>!B8wb%wSH6>Nu~$Mfu6@dswU-P4sn4WIvT#Gp!^rp% z@=5hAvRiTPzdy6fPOQ{*wsS|hygZjiP6I}MIyJt{%uRR*3b#V?Z9qQCywXp!DpO9F zY8W9^`PqKr)$FVPYd+7IN)6>-1a@Av|E*n}r=&YE{X*T;?N2p}e3xdn#WR0K|1LAv z(~xh=XP(%KFGzKzuP)7!nBKZ~-_$G>UeX-g1AbA@kST06`_@$ftSh`c`;7STVUSVo z%5agM`l@rmp6)^@Ch*ay{{~i~wwjsIy-ItNI0sHc3NigY8#*u!v0ecV*^3ik?N!hI zXiDGmHO6x8lAJcSGmq4owhc%h+2)PVnywL=G`0g3(4V#n{3E76qdR(H8q>E}S)w~q zqO_*39siLXdNY^wc1?&kA_-ki@Eow+T|U~IcwTjo$H_&$73I;b_D~ZRXBJs`}u8$EhwHxZJ4@&qqwLE)N^vZO%ATEN)YTbn0 zHHM2946TfOtgtcLwyRnIvmIVel;8E%7V7IfJnm7xv%LIvvYr`q>G&Ghcil$9VqZ$Z z;{rVT>f~k^w zvC=Al=**~1xcwS-*Q!7Dp*B>m6Gc>eEHs`IlcrO*dJZJyc#ob-3x%uZwuJFTH-xZ*feQzRD7X0x~-_dbfm(% zIU$EL_#Nh--HRf2nGM)qwYvA{3j>)n9$AdvgkGZ_XOg8V3$G&I$un?wk4*aPzpmS>c1H#5Pcn^PUlDQ2zP zSTyb)z7(!37i*C;*bQTL^h8BJ%_VchL&gH@s3pmaMStlZIT6EfE*w9oDn-vN!b<+I z+znX^fGcj(P!f%MbJ}0Jd|tx&z~b}67FB!k)w`5y2U@vvF}vk~BXtClkxwcm5HNDB zBmTriusg9{We(Z8Tw{XS2v^+NZnBrPntQ9l~7~1R=n;Amy&qr5>$uFM|pX=nA zPISfS!dmV0U>~#WafXMWCHVV0j}B(8+h$_RPCzxQukLI1onw95@8@Xnz2tZH@g4gN zChqpS7{s7cEXVxqpr|hFNj{H^c;ZRjJRfd9(v)`{A6g{uQ%+s=1dW-zUx3`W%!|gcl}kUI75FT{y_S{tNqU*8;wXA zHSEdb*%@N_CiiF7EM|1Bt|uNdyH6s!UZ2?F&!Lm{kzy_M<~uw)9(rNpj@iIQymp-j zqf?xZqTJ{PyDr)FUE(h`8c_I+rowGJr*B2(F0nEG#9cpjd)XXVCeu3kQ)5)AQBBxO zoxx&nDO*V|l&JRIxY8x|tqkbwF$G#&i44MIri45dnOd^elaUZ}HV;-cdcnq_tB)a&Rm1s6Ju~T)UU!72(g!N6SG+~64$<|IS#>r+pR{zi%4Ld^YanHJX%$}Lf_&xO}qX>&t zYl&)N78%y4QHmZjr^&(8iGXSnRFus{Eq-R-4$Fg_8w5q!NH=VIdbw`1!9iaXmRwdp z#MLNzN_T4J8gq^%kV8YOPu=NjSW?gZ$+nQ=x)d@UhgR6t-B3I(g0>d%>1wC@{;It`w`PL#*x!TGPXu~84CQLx!N2<*(MBAQ(UOQGbyI%KYSRW zC-~^%ee`>@AFhgj&)a0=LKv>?8Dbe;{a@NLxrc?R)V1?mkCZ+^-R-+jeC+Qo!7sUD zYmnfc&-DdASz}Pxr&*x*lDH;(dCL~#LwfJ+YF2IdfRpOwx3_)_efq>y?6Wnae6oyR z1g-Gn63k^JqxjH3pNBzSri+sa3gsOBG#?DUFkwF&h07X3(Q_s@GNe=JrLg4SVo8x; z%cFxhmE=G|%jDZH1`aVg%ci0D3C_kW7oE~yo3-^p%w?)2*Q)Sv(U4ryk7H_sPCGIW zJ(OE;Ray(jpn|DLWIux>LOm*s94~~4Sz?AFA!Z;&%@g&4J+YbjvZ=V=?wYXRf_SgY ze>^N%Ik%Rk)wznRR&nl)&-~?n-Lv&+h@!7Wuq^QB7f}KP>xo3u0B^|=g}hXo11g0t zlLYRPZWy;l4`{Ds>H#}J-!M$yUnDfLaKiR|oC8s9G7EfdYbMh;-j8iHCrNv;(vMIA z$W<2cQFOsJ4U^6?=T)3qNg54G!*zM;#=rMw(cEWZ`)#n#qOK_fDASKD@5AW^>*F_b zf4Czoq`7o)LAAIuCaxREh~yYh$azde|4QT2U`ZSEGH%OuiUbjeSDzZh9*1|}&%`av z$=9g&Ns*?4M}nC+qn=|#K!1&yy|{Xg-IfO>KQTmzd`?r7{i{&>^{oN{7qLr?+>J%V zVtkPlkw|@)Nd!?d_gx;tkYuAQ;J!R~R`a>t1G}z@6^z&>m3ZJA7;j{G3qxg!mR1pow-*GoDXOu(0ZYQ z?@m~JiWgocE_@*x8I>;5<|%y*8?}8%zvHR}PH8+wfIo0z_@Tb}+)FEGt%uwp^B_MG zZ)59Q02BS)f)Eh5TU!>|b3neGhKW6g?*y07mVYkgRo4c#Dcj;kWy+?RAL1*DZzh2u z|1NR8OY-@o-vMHeR~^3-K!e)1JVtu@l5QQ|k@X!ElwG-T)Hd;i)Yopdr1j%76y8km z*C?m^^IwL@$p16R{uB|!vhYucFOe?y>)Sx(Qj=Eipl{|)t$5PnOaFef*R|wvPjiOX zc)rl~=B4aAT1b+Uf&g{EjNTxSG#CVB!6nrwbFQ64R(sC6%x02jPpPSd-g*ufSjxn&0Y>q&*jd9og@ zY*ctcVk0Qs%p^Mjk6?K)#tuxH(VuV7+iT`Juf9%ai{2Yu2_5Vlj>^vs5=6uI2NCRo z;^Ndk?CJ6@8**YE$^5%=4-juPTk)9GQ;KBE_LPXbzUZQXJpN^FnkVNzEe81^fYiAz|t;Z z;=2k2>JnrBWuo{{oEi9J3PlB2UX-=?Dqf9z&boxyp`g$=_D|eeiML4WY%Rul2XEe@&J;HT`Z?4GHpXzX~Ii z=K@gwP@<0e3VJ*dXW6vx{)k+XnlMCprnn7cT@Mx**_DO%t}kmHlc;zzovvtn5wC{< zWq6i0PEROhROeH(t~(2O8|ETD>&`X!(NEeYQ&0Q4inx4Dppm-!DKonYc$q$CXX<0E z<;)gxCAbvViy>-{OQW_bT2KzI!d*ug0z$~h)_DtP&}?!NH2%|}UM3H!!8j!U_a#79 zhtjDY(Em>IDz9+o1IcN&tzSqpTRWgzHEh1Sf_PXIPbN4kJoLJEeF?xsyZz8}_vf-v z_J6@eI5ozY_x|CA?EPj2&KJH8(^cNSa=3caGVrwi3wZku5V30iZJBtwE~@&^8XWPz z*AA3%OrQOXa9g!)?4Z?I%>M>E{`TO!B!#Mt@`Fagag@_3wweJ^6r9^TdS{2=r*qm=HbY_p7fSzGkQP-c9^AFzsnQX;TM{~tZP#nJ+fB8F+ z^lIL&IDrpi<8O2R5cX+M9_0l;25=;1PX=^zDtei zI9{J?&ZF!}bn1RNwQmqqny%>0LrQuq!f*`7)zb8zQ1=O#`^waw5;ZG!Zirg^(s0F}iv*@+*yUNJHZ=Hz(KRBcF}C?-2b| zz?l7H=C7;|gT_&>j1Kca)z06&Rf+WPncr%{!&EZ+6S&K;2M`nEdH*f{?2E;DTXErs zbx!1jlYT#v%!wdbBC-gCX(&i3B1V^uT4kA?_cCp2_-YK*#?VRSGmRPB|Ty_9{= zNFl=C#h1o-mM##_L@`A`W*&ST<|move~-}OyLIICY&uozIXCmyCCB#5bwwi2aI3d@ zv>wa98=&lW|4yyv6|A}#C^KTLKHj3*z?dY#JxUa>c(`6x$@p(E^Tr8PUSkg+PMu*Nrne^OaBvRj*pJLhOs%CklS$DU8U&pBIUG`f-KeB^T&}bhL5)L37=D>-%fu z$tN*eQ>Qulh05V26=m?Hx)Oey@7wjY*K#BfMD0c!bzcZ+BA|~&c;3s$Tnnfv3=XS; zK>12kcPalPxqq<57agaoo8iuGD}mN22Bmws*wJiyk=FYt zw_q4#R%jh7X}qf6Yc0Nv9Rq#8@j5{(G17_^1N9`SAZk3JAWmuAK}E6pa}M+Gg^Ln4?fhn$GKMvUS+>>K`*FR#CN2YqkqGYA=sT^ zE1Z+`MA|paN#QmgBfixHdPtp$83^APjlG?eZ{N0AvQJxiG+}K75ofEIv_9qV?ruJ2 zg3DEO1~z?IK`Y$V;PY>HkMbKiQ(t zD3Lkrk|$a}b4pfgZxMMvS}4t5MZpUaQsoR?R<+8Ur`jcoCyzl$7Y(C`CU|YF-E!KQ3OIgl(VF@(S#G!OKOCc|8 z%ZftL=}dyQ_pNi>_Q_@NuVg5uMSjiGtQu-e-cBpiH9BJ(bpGSlncBvJzPZ|zzcxRJ zpsluuXzTVwPRR$m=LIE!Uc3%3m6CeIVl5;FSGj*gx7?`ZtJmnIXKZ#p*BDCgD-QiL z$5#4A?Uu)kAN`v(%lXadKY*8DxHE|T&_GY=5b83Ra0+k{v+W$qe#Eo_kUk`Re5)h% zL=Z@7+;&7RJK%d9#eT&H?BK70d{OSK0B9G22Y7aKm$hR|K~IV{2f|4)FiA-XA;LL% zaA#Lvh%LO2&rs5F^m}{q_Bko3U4Vh@(NB|;4q^n){eb)6QPsAzg}rKlzuwB+1pD1s zOj+RSA`y1V3#aX+hU`Kno zq{-_-rt^;h z5G+n_rN-Yj;nw+Gy;hI^a_rMHOMq%Pu5LqI>-sSeP6UCwVwX8OL9Um83w z`k$Z)af?SEGUwZtTi+jav+MIR-9`A^c8$W`t6?{N>DpIWKE_e-(#b;z5N$-WDlPY& zhMWjbTs{VsOSxsoQ`G1&fwo=Plx>R__FS9353N^Kc|Hb9t*=Z5`t$)D?a@JnxvTZN z&abYQ0-M`ZLZ`x$LLMZhq)8`)Uj9%QpXe0iTm;-Dv^-b&YK^6pv01$$XO!40T=s9# z>>Xq}w!0tk86eNM@88>|{^%<#2wqI_nTy-h^a(EnFhl5jN4@9Wr3B(Ncr9c636*1k zYQQ+po~Tn-@FeFoK(iA-iJ*`@z2ayR^<7+B-^ARniSkIqmg=gdRZL(LXJzfUTa)N= z*UHM`;g`SK&bq$Xkhz+ae#?ch-WIg;zdqMS(+B;YAuQpLXK14Y$k|dj|97x%(dr6Y ztg68veJJpM0!NsC6aNPqn(^ENcnEwtbA}e;`1%QuWBxxd@8HWE4ta$9Kf(*x5wahY z)i(I4gW>joQT`~Q;SKmGp#?WVx_7|86}{)#p5=D)X)ovZ{F6V5Ox?X>%WgbjWeXNpqY{bpJf&~1zwT1p$1Q7l8qT1#SNb|AlP>)W* z7N35r{Q>BU&@(EV^Ifr=-G&k`99@zxHD2QQd*30PG-zN;omKgJAL{epV35<2`rzFh zS^WWbCWiad{LJHSQAB^XjVwnPdDbzSqT(IyML zX7XEc7)xU6x8Q{bKuf-NdRyWaB5wJ1NP_D#@UjrLeytY;61N?wVFgg&l+v2z+9pT; z^)%e3@1D(mLs9lK{Uo^l6xPyil!LH*t}rOZBj=eg73E`4)Y5;#1J%$(PL3Wuuo!7R zaE3_O>y3}s0?5GI{!T&YG#!h~(1$!ID}Qk(K`lJ|;mywT@A$_3xhP*BA>s8wAw2pm zJGP0n;0P^4vg^PV@$HI{u%iiw&UhL)JXaQ296tszv&p&EUufN{>63@@mA)cZmsR6( zDbgkWC0o?B<3Riu&dS6yzzbi70DtjQ@i_WwDEG>vual6ag2;sf!EKGrit^H&BgwkK zz)rsjlKoq=@ck^`0{Uo{h;xQSXI4h7h8iMWY9&K_>lyLE(Gl!kCs4>zGqar4DZ8+l zqVjw6m*qWq^KDbOpzU+afj3^Xd`i)qqr?zePVo}G8bu%h5{@>erU zz=a!s!c9Oi#@8o)S5w#KnCcwlzNCe~ZEWs~7%LdSTrC@1+dD+*HD4Iy`SJ9K7f8tP zUuu{|Y(Xl1I?_N4FY;PsdF&y0)_i1<5*^c(#;rm z2)`M@&MljwXq+SGTP7NyE%C6_yI5{W=`UdV_g`xUxz3!A5SP8S@W-EYlmgLF(skVu z#cjsCGUH>3g5>bHdlkvJeauitd5NR{ts|OH^a{olD@;m0=?NtygymEletNS|}*$z=@}4$v`VWm1Eap>VQlkMSRc*YIN_t{H@jLq(5~wuA%4)@|)rCeXBn6RF=5UBZ$KSVzO_6NfX);wup9I>@* z6OwD2JlR_lGTESBv}`FY+B2;m1D+>n*=>Vyis5&b$L9$O!T@TQ|Pn$c+V>zeK}e}{hw8F$ckt6)3ar@!oB8d>pq9uxGpYT=8xgcllF$^ z3DH&3<-e{j_9i(YFPi4b*469%A?REJg}Qj!1%PRlb8YVwC%!k#liv6O{z*60T#2lo zb=@g{+X_+9A42ENhZSetFI8rlY@p8a?m4jy*;0~UOC6G0=@n_SVM{b9bbnd1A_@+k ziC*>?T1bE)@=XUHyrnW5??6h&dBTzd-<@BZ&d&u*JiHBV<-zZ6<{fF?#1C1nMhpaS zry$r<#}fF9o?nRL#!M_3zl2Q(%*Fk0JWiEEeDF@#L?_-AbB5kSd*N4C8dQvsDSHkFpnbwMk0{kpU_PZ@v`~u1G(4PLVM))@I#DzrG}% zT1@yMSG$ir^p66kMF_^5Aul~hdYe9=I7CVewd7wK1SGpJ8&(=MMCt%~C1!aP6@ImI z8%RP~kfiTAtKkG8)ijzn%t(TM5pJ8+HA3M%(HW&K^{0_$5|>NM`K9vA6#?)F>)a<)VkXC zUr)hqexiza+jygR)TTk!1;KB6-&kd)`@p?+TPlE5>>-4bAie}cT>^GQwOQ}jP|HCq zupe)v44q){X!;iynReC=zEdpA76|K@IcFp6nWHT-dtosbTH9NjEngw z7v8IS7{c6JDXHpPrDT(ti*bH;R7?;ZA;T?#g5BlI8fyBImKm@5A_$+()r`s=^}2jei0j_)S}je$m1`$B%)K& z;;x;($3vmqKAXbPIgj5j+|-+H3l-m5t^5fbcoUYwv2PY4W7tf~N%M2OSyXXD97mnm z=8|ERpo<6|SD=Pg^e>&LcM#Zw>A8OqS8m=*dF5m0xGukGC{!P!CSJ+|J+XfATN`6F z>T(uX)slTRs(QSJmOXK8BaNck23AOU*!0t{NUDwUE2sCWt7Jb_o)Gp*f}Ogm?2x_k zYz4*g;rACfM+~WUc3pBkb4DsX>v#-1>KF7I&?Ibp&cKN z72Soou_n;Aamq*Pd-v`9g>%=7HxsC0mFp$gJS&y?W|_8g>dlZz-26Q|__Q*iU?S;U z;=EfnsbZ-~)DU0efbN_%@@?4G{xxme(G=c!(HhaUaV)dL$pdSj7k-t~<=)W+iZ0LV z^-Ak%Qy1RRNEsU4n5gy}k+#?u48F#wSp#iYzgWqFu1r^sN8tJx&1&Z2gE+l4C*8hj z?W7Zl=(InfnzOK^H`2Tukn4V{{*xMqC@zAkoh>kb=TQ#c2PoM6{xt{90 z3eX!oHk~ZDQu+sgPo>zoHKV3+|;4UjzJ04F9N!Ry8_LHPAh zzoabH3DAD29q(*tRpKZV-uiyOe7)w}MgZK;ZvM$--*TL9o_cUjK0kEysX@1o+Mh2l zus^83d*!oyZwF`lg0Mr%`d{F$ecLZ@>UyF%e4Ua?5cp=KnofL}!Bm#tACqwjG)&%Y zk8%Qzaza1&tH!v}I8`>Z$^sLwxwqF{z-2oGw)wdduKLD; z3F`g0EfXzev}YxycPWsZ3~d;Ae&%O4J9SBGqwpwfrV~@Z;Qv#LwnU?Cb*jSl9a7BM zNy5+F9J(D3sqT@AA=8V8+2uqc>b%jg`uNcs=ix=bQhAV&9|}SsX;Ts73<_*lhAdJmTpVhiU75SM@T6W9cFytAGWxmkY%Q6K64@8OTuaXj9`QYAcYIa4g9PG5md`i=p_OA{-C!W&C3(0*l_foQ;UU zjY5V8&sTSAeZ8fF$3+;s`U+639^x(M?%H+kz>~r>_rJ}agY63mfOW|3_!|@w2-n=;^3md{x z!9mu6HRiS%$UUeJ72YNtCvtwoiQ_{8#KsZ??9c8Ho7B3uCfdPpqTlFnB@ofU4LX1E zFVZ~`hggWB(;x#$yL~B&hb@^8Ol-y8krTi!hV${I#Vs50C(1)#U)-++4Ccpc#4A z6608BzYK$G?IyS3#M+yT&Iq!Xqk^kp#+j&B?=gMdC^2NpBP}Aw%Q;s%5%Io8v zT~PUq6T1#&PmWzPj72HiK8Si|R^DU|ecl7e46JrU^!S<*`;fS(%eT~_{d#~U$z*e5 zHY~!Jmu4K{IHSbS_7lLpE(V3nCSb}5CRTf4hg1LAeUBXGw$b`F5oSU87{+!?~lvhGXk$=q}3o{TMuW8xi#OpGQm9lTE! z=_T;9>YSpF^|1Go-)+QtWA)AH0jJFqDgL^#2e66T1?(N59!njBD{u8S<7{3JpfnG( z*r$ZeQO)nf=l+YWTnD@udn>Lu_6WpE!+(0m+Yi3S6fK|xO`rG1_2o;dQ!^i-v=&ln zYo$*dAuc!b6SC2L;at13;vO;MbFP}Q{6BDAdu*(S)nsI%C#Hr8X!Oj9eqlkbd&Bsu z8$ulJup}a5H;{p^qam$*Nmqu+{{JH4sO?Kh9p7J=2nye3aPcDqFmcIAFE~Yd% zHBnV~m(5IA;60P6SV0okRtgacW42>pc-rQx@Sx{2nml z4T0%r4g=23eDE#4*n{6q~qq#e( zA~q5_zU*lTu+19zey89OEMb2b>5K*}-oR$hygZEM0`@maVxM-@a8(LG1qCBDWoh zmZ~xNFCFPlRI3Ber$g4#`xatGsCHyxMNy8itwlPxi8_)`8KmXbY&dOsV(qLmDwFExM%hda2$uR9+(=$Od0ek2*DOaFIcfi(Dmajo za0$_4##8=Az<3fN!g95w!}#<-!Ai~<7{+26IpmZ+fBm$XNa;j=ju46__ItK(kh-SF zD2w*ANOM|uwMp;K!A0l5T6FvwOify*CGqvm`buPaSI@Xf8*Jm(l7k36hPyZboZ2m+ z&#bs!TlR0_Zw01!toVyiSzDqC5P>gDB$QQsmgpyBA844&?OwJrtBTUk;Thin#%T}R z&PBV&XOcUP9dIhRHAVRA4ImNuXItye&PV?Rx@^NbKD-U2zH(%^5jPeKYD z4Ds6lpSI1sKrj8^o~2!`&*b|*`tz_N`WNv3%2k$eC6-+qxliC79VPxACbj8$cxF}| zMwMwgy0!-&S`|eX7MmMLOx&JEEy5s0u}UN#Zq{jFuC+a4>~d=|75Dj(c6YqChm0biaWM>HeTfxh97>#qHYA=646@C zDjW)ZJa9pgBb*9FZP%6S{oEJAq6u{+emPhJ`k`}5174MEPT}+LDDUxInT}Ma^r)Pv6v80Jj?u*%kHKjwr<#V7baucmZplIe~bBF07yW$ zzlf+`ERU=l%f17LKc9bT$-RW%Z}U;x?}yGfZ_H5m?h+xN3_lca>C@|2c2+TBWSc}7 zo33I^^j86s#`8Fbf4pG&6$BCHxiM7#Jpt5!_3o5K<}k`~wgttT)QcJz2dF^-k0>j* zTa@*XV#>z+6gAi|o3hou$+N91;_)kf6WGm|DX_bEfTzL`53J^?FvOKJcq$ArHHN3c z5aYj~96ldQIeJ7WiKh0=Y4suxisZg(!g`y*2MG3 zxytiQzrpibo5K@LJi!yEU8cN!vv`v63d+YOo#!iAOO4h^=1Hw)Q_`9=%1Ne zIU}9UF}{xFA4^+U6j6g;aX+J9(mWW{WglgGm-IBX*zi$9{mcoxj?YY1|K16seD#^- z@R|1@SJegQ7@t_9jQW|W%ii*Fr+BgTylnu|Zg&_9q_62XW-LHJEZn29-PC&D7?K#O zTK2s3e{2-^E__@66>lN-d_VdlwP6ZJWXE+ab#Tm~S#>HSOR& z1$7*;A9L#WQaQ5sDs1GV?dT|NePYxF4P4leNVt?W(yqz}xZA#1%sq?3J*b~B&$NHy zUb8pRqG`^!7~P7ga=K$eaG#-5fv-gaGrIdYF#7&@MtXlF7*k!3`xTe~|D)@ffZdHC zFtZyIw91GGUa$@iNqs_~3Es@uu-}Q$Nq(%HwB$LSvBtA~jmsay?0?J0g?fLWx(2v& zti^Jccu!UT+(nO!I_^q^KE`z@Uq+D3AkBvIHV<;PURXofctr$SF_*Q}tN6W$=APS< z|MVK8wl(dr!`L4o?T6#S7(e!B_^D_|=;~A1S!wss<8(|Eawe%$f@<$AQgr7yX9 zG$;!z)!5S;>e1SC%UJ|KCmY1Ecf_#1n;B$my^q17xi7Y|*jrI*pEcKHw9NenU%AI7 zJ~c*63H%JDlwZX)f&cwkJoWqyv|6j4TYM5vHRjwI18R>wcdo~Y8w$xD&dCjOFZG)dr4{%7uji4xYAbM6 z4@6oH*8?$rJ>g^X3GkK7<$55jFhW|BO!#Se6Mlt$gnzj;5wLFv5m+DwK}&lO!BaF* zHlGCEB47Thh3DQdxh&e�q~d_(@Ib00*jY{n(S_1X>j5zg)b7*l9is7lEJ z&{?n{Hz%(&liz|7vvLa5{b+-8O!+LARNoV{;KSM)tvMMeo1tnYg_pwY-7Pa2#&SgbtF`+zvTMh`u!Aq8h=x+>3=7l8Az z$CP=enu&b($7ufYD8PP~cJ`xB=Xgh3vqcXl<6X~{m>2xN)HjgfNo`e2c{1PkNuuWN%^c7w_<0*d8doOb-U^jj_HITXDHIBJz z@55Yu`V24WaS1QET85WCIE0rej^ft}?=sgnmoPUDW;68Hxy;Sg-!ZpB?3nT|W-=A_ zhD>GRYP_oF1Lobu_2W#rxPGMR`;6<`t%sBo{CDN!efs%Z>C_H3FCO|_@dw~qHq#)1 z?E&!KW#&v}XUS+I+fk^oY07aE{ZEx9jhjTKcQ&GCB}K>k6xdNQa(f7f*9DN)WtT#jCWb2<98IG3Y8$Z$D&81K&r0bHK`D~8L{$1z-<-jm_-^gKqDr~lj*SLNwL zyWy%l{fK+G^NU(sXs*Wy%P!(B&+ag;xd(B#l6=N}TM^@tvzPHqKfrjcUB-weu4cq( zdvI^x8H{B7QryQThVhl)_-Gw>Mrsv`OKYNWKP@NRuh0$mFE_>m_6@*We6Fy7J#?hG z>gw8eSTFGn)@H|AcsqO6*#~RgX9?B_NJxf!rh1v*f{moc=0pl zH9Du=+9xc5v=olN0&@1gRs9p|>xf^B1$cgKHBj!f!Ea4+0kDB)Jb^~?-+U@+i=4Q)I^JtODsrc)4VL}$9J#b$8AwQ)4-gc0Fhktk6 z_N9g5za{V2Ngp`=#k~*L+wza=+tjPxE{CRs@Af>T0t6&#(IAoS0~GBth*XIx8k+<1A zZS}iBxqotv7Q!{U06BXfpm_6s(ZTtdw)Z1$SY?qBtAj_=12pK z?FDw+WwxS>?bCTT{au}U96I#sT*EjUa17&Yz-H_{^vgMFXzS>9)r+J3+JuF6OMi~K zpZBl!=U|cWy+VHhWAQ${9OD}xf96*G&KUW?lGOJ76nfk_%`N`vM?~9RVt?L*lVf|~ zGY>(|-tnQm#AJ)Hl_p(GCEXd4 ziee~4=sV}kMA6Fbvip6%o%x;LGpGO4JM%v0Jm;L}d7twf9`f^}|EYuPiJ<`diw?2c z1_Yrox=FqSmVJxzYPNpjmB1o}*M)XIuT0+Gw$1-{?EzcYCst(}1F@=Qy=Z(1-4{+X?;KSgr_)KR`x+t6d{LF{(pH?mB^=r1l z@>36=+qP-jI6rO?Hyh_#vn$`Xe=KaLdMR-30&pDY7R#@vU)A;>+`l^BNY}!R0r|6l z)aHGyyYboq|v(W=LCtsH3jk*l-+q?4%`aGoy#;ud}z34#dEILS&KW=7p?S7#3Ji@3je6JiQ@jXTDMb|=gIH$1Ajm>AG-4{ zSoXql8s76+NPktd??_1IfL8prP1hf&zqOaoNnQVfpSEdRT!SS8{;oVo(V61oA~nqJ z{oDOz3VDCQ^*^osS>xJA{xWA70bJnv-^E7JXGTMU^7|Vn&a$JjXF5`!)PwQ^GPIU0 z5@+y>rH)idB#_-ZgB&juZ<1NbT>+o^9d6J*_s z!l)iIOWwr>_+-k|74kIs?Cg7yjd4lKD_YdA#=hK4&s^rfG7pwbP-a^p{YgG2Y*P7} zwrsQc;CK3qe5~ZW_~bnB(f$CJaCje=Aw_2|m_}*@D}CMl?Pwj2>~XYiMD{pZH#XYi z$d45ylKZca9gh50$PP#TD`bZw{}r;sk^c(W;mCi5>~Q42D&cTSQUv2RYs;)`D)~+1 zW%|kenI9vKXTv~Qnl#8|_PP7fU2YPuV2_CF z#vOTbAA3}=m3Z_{v2e`VmE5sQUb8v#Pl(4&8zQtCJ6LQz^qkQ4ZYr0XJB@9ZJe51~ z!VUJMV;jVi4|EVt*&558x@IJMnrDUB{@bO(>64a-9Y)s(XABJE&eSnt&+2kRJR8gt z&VJmHJLk@B_S~#pT*o9;_PjG9@wX8Zg!6x@wns@i*HhxaUUaEJ>=m;`xR^2HejgOaUb4YmymUpC zaG7&3mp_YSFSpU*dRs=YR~R~oebn=Xz8(CzzOOp7S60M{R~3yF`lXE#uTDr3u8H#I zu034GUbibk`~x*W_~X(7?s}(%Y}g)nT~1B^4*P#MSl>f`0~<{P@%nI+ff_#Bb+c+R z#<#Segz+u4br|eY-FQ^9P)?#QZxB+{5_Jon9Eo@asFn?ZZ1m@2d z&cJvm^GAasAc;dA!-}Y02y+<7w)bO2hb%zatyRc@~xj z07IM57RLFeaem)6#`z6p^MJA&A9OT<<31__VWA1&wonS((Aoga1JKq|vPPeC@AvZ2 zb(-=^6M43IA8$3Yv;6xSV`da{cC)p#+euhX!Rx~{zFRDi$E}Wsmw~^x4SE*cF>g#8 zeqObsOm4r8lofRV(0=P9liP0%=-DD(Gr9d1j@C~qAIJm6%__^k{{#D~**eHA5tbBK zB=EQ7K#FV>>#(w^1^$l=`ovkF=KuBG{Z8d&6V{L9ukMC#3*z}oBw2(jGFYHHfHGeNR^!i}^ z-%Rh^r@&$l<+%ycK!rRtVX=dsl>~1ux_<2cbI|^MJ$4U+1+Kl_k3)+6e_=%>7vINC z{`+j=sY_}($Orh5|F2E^b|1e#3X}Z3HwvFr#83K9Xx`4B=?DDCUUrX#?N%=Z9>^a! z?zp4vg4mG?exdz7NWaj2AE;m6!zip@ z|A61NNqc=kf5zIQEzU!&#P2#iW4Uu10kz1lgU!zA2EYIQf zk$%2Grb#yVR%p4R@enUF&& z^r6&H`tbVQbZBG_!7Sv{VLRK?N5(y)k1pLohwGaV5z{Uck?lLtk+mm@=+q?Q)Nu}R znpsQ>w(1cv+qV&C_+#j^-r@8)dy>9j>PZW`-J^vKxpds)t8~2h29Y34CN71pr7!<{ zoDd!CLR|6NPWB~s>!>6C(cI#p>Bo%&=HeeLdgA}w|q zkya%F>69l(FTPAN8f{FgDDg32SD;y7PU$r(m;>qwyJcF;zV zbmI!q9dB3BUFTV%QilnmdsddBGE-yGeLW3Pxt0y5qLT%uqTYy8`B0tnph$)D@Y*X* z)y0RL>d0cwqu>nAH|hixrkA#NoUkc z(*znhrx?w|P(~|0jns~cC;Nm%lYMs{CUw^BA<6G4Qg{AxQg5;|sXuB4*>B)@(m;C{ zY1rA2G-@}UF?wdj7?+tbCb|8X{>iS+{t7u3WM28zbWBgdMdfeVlj@4X4#`7VUsx^RKnub^ z+lVmitWFrUv!;!nnbXE)2DC|TZ@PbS2f8)g`aqxheU7~g@R`a_=%)R+`KzTIV(5I3 z#(wOjMgXs|kRtoGEt&_?_l1zzKI@21YA>4RgGK7CEdRdB+LrZ7@%``SeXsfO+Aff~ zDdzoIu%LZGx;vJH&;1{hx9;cny_dhVDCdvj$(zY}1uQF}JPtw{A)j+Y=!28iQU~AW z(jg)B)S=Ws>hOAs3XMz#%tB`>Y)2h+WL!CQbmLW{(9|M?`>>xfiLFf@&&NX*8AkWH5E@?sAY8 z>jcuOVr1!*qb$8R24w8B1{sNt{7i3wEHgQSy6$TLu1ERGBo09`NyKR?YgPx4wR4Uv z+tf#vy(N&!am)cX?2>{YEnJMUUPz!Y3(HV5xY)KJ5##DD#4XVdXbz0TBo@#kqNMQc%ev10bSD#-0@Hwio z7UxenZQN_|+2ha!0xY(cg2f(+vBD9BoWjrJcknhbA%T6tp{#B6VejcwXs$BB%-90L z&Rfvoql~EVI3MEb6itva+K5akR1>5cS~IDMhGbfM112rXib*fdATxIMV>07Ulh?m* z&qzW7$*f5gO!k&&CTGAk=7!rICUfe-8W3RMLtu}Nt3MX zeUYhr@S1$^JcN08?G{;e^GBvSQbayF&to1393f@D^sV_j;yPhz`T50PQJ=9X6h@Hnb+|)jr+yNe}UOlF`hrpX@O<@vVxY5c-biPPqZ zk}Q;2$wNkouJ$TprKme{QW}&5;<}NX)LW&3Yx#XRX_t}(=@%L}85SE@8r|P=G;-nv znop8Qtw>jvcGfYnPrw9L-(S8J=y+Lh$Zewqx)aqodY(E0eUn$5eq1Gifl4vQaA;qa zVdW)`ad*Hn&Mp+0$nLWGpNSU@kR-7N1|1d+5*%X<_NGKb=xr>PgR^Mp>TrSS$n_jE zM}L92mJ?^#a8JRodTWlQ#zcW-kpYMOa-d-NuN^oe3i?a_%iguX#gxAPx7#+=&`C)b zaw?Tcl6weINwmbq=DNqaUpqC?wDz~g{a`TCF01I2G@*2JDMBu*RJvl_mg63aUi|+5 zZAJTk&%85C8pPUNzyIeqpXYu0Y$)Qqj{67o4NK+vC5nRjk3CI! zFEI%k(0eyEaMHsdADgw5?|`x(-)EuJAp6XqL6=5QgN@IW4?Z!M3%H+NK4klFZfMbl z^1!8?HTxkojGr3L6`TIRcd2X7PM_2kKfSpp+Uz<4%t87&XS7N`CnDl&aleQgmX;Kk z=Tci7d{9taFi}!+U;=YS;Vmg;zY&zJbdc!L-i&@TM}a|*17q;N{TV|Kea7&-J!52g zhcVi(#~9ZXGA8k)KzQM{pxGi_LGzpff$79s0%G${fmy#(f)##+V^zQ~Haed(Hi}t{?e$Jf>y?2_n`35-UDS4_?WSA-Ic7OyFFnUN z3``U_&U_+h=deuR6mVAHtUFEMBHAx-DXS8Qj5Z5I#Zws9d*28;Y_4G3Dt}-)u4&J> zXGxe&5q<)X=&?-a36q#E;a*JFzCKL1k#<>Zj zyke%$rRxH(9o2%q2Tut4EiV%E-?BsCJvCJ@VCgYtpx-)y&*bflulp{>&u2L^$Y7+v z-*GzQ|JzQ%;JaG{KvhF1wp?SOtRZZnr>gnpe%#B*CI2dHgL_``U9oXp-7p8>F5m8P;o7O@aYJ0E?-Mk@wImzkI^=uiZ^rd! z7}v}co-fef!M%p>(fWT8z=3j|Y+2k}@vX+nJ8?d@HKMkN#0WPF5s&7vqV7HWE^Pm| z*X+A{&8qc0!d;)#@PEM{A>L{9#`aIw0zV$P-;7n%~dk^HN&G z4a@O}%j>eNIQV2taluUek^?g=&M1cJ8<{?~FxubT!bT^=!bUOQLgX~kLX>_q#xHDd z%pimDG5(I>7XFW-ECTd`Edol%S-gBk$;KI~>owOhzAAexDaY5jQ22_!-h*jgFrw+` zT4y63rn1FTl}k4OjC+@U{kPZiU!i~CmIfZj{}uYzP!CsY&uiCelQng5-XcUx5M4$1 zL6eMmb?Czue>UE5y(b18HWx|VD%VRpN>Zh-;*UiBZ0?p?@8o3+vL;pQy)?jls41d$ z1PFCO++BtFX^8lJPfv^bX`H2}cdg~!)ibQ@CrGRc=FG6s=`XQS44Ywl-CojqrN@jm z$Mht2QS0con|9TbV`$o5S|D*4D1(kOZ`QVRmt~`Geo;IdXK&KMpDbx!QvYDY0^lXri4_%{r zHZu{0;n=Wg*!r0E#`12wRB3Mt7E86-o2a-~_9ik0ywTpg1>^2nVc6b8e<5vBu8G>- zG$q%Bw>O%(Ca>8WvE_2BzsKLNtbRW`1FBWH*p4U}(KCdLKR7_QRN-O{qHyeAkLShZ z^(Bgf%gEvavApC!ID1AhN^WFY#~SUoVr_K3W^ELUSdr6sR+L^u`h^`OwVe(7%D-CP zdbfX_ql#_cRyRhA5V;}hPXHRt<7s}q-+&KT(4Ps2CLx-IDDQlo%El<_ad*q2j6Q?z9+tCVSLjeh$bS+LKr!KINt}jN35_F6?^UyjJvo;zBOr| zd|R$u9=~jpywE62UUGLRab{;Ld1;~#Q5M~eC|}}4RD@d+mBYN{&5|a|8}DUX$<06a zB|gFZwjMg*L*D8fQ1yB#&$X>}Hz!cM5{1{g>FWdtv*kdTEBZjlwq!SH{K4O8c13e( z!j{W)F;g$CoMZ%_)lLU>)dBGNg#hrPumh~m=>YynX$HaOW z)F9{){h9Yesi8+XZMfD&YD8qwMrqrDF^r*2xQ$Zb#SGXif4;PNZZtIA84ZXZBcR!; z2+$&GG&G+%8d!|+ge?O+fn_fn*h*vrtQ>mM)`I82M%S9Qd18h1u#RqB@eJwVFSK3e z6=~aqqcpj|6xc6LpdCg%29C21!*<=PfYXq8=-hH2aB*D>UFs{OBIC(WbTwP*dfyLr zIP$&Jt?FyKANvfkB!cB=jjVT7qNiu>K_8Sjl55Hcj*m!bSa^G8nyX;y|Cvsn9F^xwP-0b+F(1tJ40lH1rP5mkwCAmL51fQR*`#l=khk zOzP)5g7$lQ0Qh&DtIjVA`LO5v?l(}*<8~``4RdY#(7Sm(Q|&{C^+L~oBjWsZwf|4h z6aG5iXFQtL$4b5T6PAm{1I6uOkFc%G#d)4Oyx$6Yfpx@=jz9P-#jXH!!j>~sF$3vJ zs*ZbB8v^U9`*6=M^oB1A+i>+ct>GUEe2O-KJ|h&2;!*Eysy!Q z9>*!ewbrx|kwh7##Xw`Yf->Qj)5431T(kV?bo1P0oaxS`koa*LXSQl8Y!Nk>GoR@X zEylQVEdxZ*vX_u+B{G3l4kF50@DSSQnou@RjA`5J52@A_59l_>$|$?cQo3zICPgj) z(0=ho%3;(E=r}8#YuEiGbQ-dOb8eXoU0lVSOZ^F2WIT!!UEM*u-tWnEII@ytNZ}8uu&y49jSZaS@wf*>@V_C+5ck%>#9^5M?9UiwKRqE}C=e zMj)DxC`vny#4QooHTWL>olfYhNwGrr2eIeQkB_^UBix#FRJcvyB#hr=DP-jXWC;su zg^9GIZ2Ra-!lWPr*^VB0!epm$GKI@AS<1wlvYm$8Wjh~@74EvbT$bu`Lzr4HU6%H6 znsARIK$c$mqby@(2U%vOoh*wUEXYtF5B-R6dnl973P>e66ShKWVtov z!b4VJvO^ah$nt*QAv~N@Dm!xNg6!z#3|ao+39@67XxVW^vF!Mj3FA+MM9?P}ouq&A z98I61euD+};jqv*E9n1GJU8h*Y!qHdc%Sy9I1NITAFm&S0l{-#=k!%|CHfgJ<3Z)? zZLy)+7m8g8AYP9+Un^vNcEDxuSC1Qbc~(hwjK=ucym>oDy&lcado^FL-Djiq^9J{s zkLVJ**K3IL{hYjaS@}ia^XNW8E_iEW=xkC4mHkKm$BM>hk-j+(yz_)Sq230U>j=m zZx&7HXTVSKnz-q&r#@e^qiW5)sb?FA>j)6hp92xR9qt*6X4<*m-s5slrI(+6GMOtP zQm~EK6$ncct+=v{GwEyQ+o)>?jDw6GOr%V#{!N*j*#m@!<3Y2eQPSqJ2~uTV)vV`) z*_4H_#mHUIe9#gKpF8_W*^8d)s>~+V$Gl9$$K!12rBw&sD-MpJ(AdW!u3Fd8$>ZCH zAi9F+2BO=Do}%xyE7I-GX|>e}GNrBlyT?*(LneB`!K(3eY7$Ogu9us8Y8^kVYZjVz zB;qI{G&79&6^&gD;(YHIuMb(XsM!Als0=wGtxOnBR86*(pB?Hb=kvnOO}{HYmm&t| zMSF>#-E4>pMn4c2b&ts}-jfiYitkg8e`w;QTGjaXZ8*|;8hmbptJtHP z_mTOrw&XF%Ci3_UhCC4xMV{>6iT%lQ9C@nuF|r`!JX>(dh%FpqO*X>Pyoqvkyi~Qp zo8U3@smD!f?98n$)T|$zrZ$A{v;O#grc<+yN#6!EdTEKB-j25jL@1`1k7A1XydCIy zAfr-{+wUorTN6heytbA)RQ4T}SGbcqoU?^HlCp+7y7^l!pP9=YlT74}<{~2=V4l z4(`nTp-C*nH1Yqp=N4z2{%kB`egm3j`fUd?)x&F^b})cO|y%x=AZ z%)MYXq73xRvJvOc0ps_sP9VC^`^;^#CameEvTuK)|AR@%`|C<% zAfiQRY?mUgJ_jTk5x?HHA=Wv2L+p7Iv$!%3c5BiJc3a$5Hhz;0%Pv1iCMbq_Hm?Dg=$?2XwGN*qh-it8u{-uZHufKdjJ~&uLKFof`KH8E=)-0aR!uWrYk0%AN z%DRUk(@16AgMBkx_W){e-2+#F>mKs6aoxkh-Rz5kL$n@!iqsc{kOo01+Q#kM6A5hie0T{|izQ-~&FSuJd`O%&S4%^cB~r4|zRS z(`vgkYkeZOBTD9g#FGG$K8W+@0*GUs)7aSa2IPOTP8C@?%ZoORkr!9k$x9;p$j@W~ zc_|e_l*w<%)pLW2x-LZJ%zCnFvgoA_o*Vc=&f8&L7th!0u^&qp|7rV;u&uAM-x2R` zzhA95;(uxRnRhJFs_^?<``kj)|2EYxNERZhLAZU&fhaSTUnt~{heuyKCW%9Bi%UmP zCHc3t%oG06^)1Jy`^#2SwZ`u}+jaEw*eSYnH5)lH0;x z)`J;B=`e^<`M-0|y{Yc4ZcS#I|NrZGpZDI|J-2he_j~u{BiFr;;CzG&f;ivlbo}zw z{DhGNw%a9Hu&0X|z1J*Ikl3vomiV|raNtuoeeg;Te)=jGIzz4vKV#T?>_#@h&&)iG z<<3^)=fyc;`EK(3+n!qV9rNXae_NK)1!~_53gv&n3SYPggl976{)>MH6&J=X_Z-66 z*rm?LR(yl-B;4SB5^eNMz!+aIW!gvfz-6&f#qnpUvq;tjF@H@AI)n5uc?3OPKZ8RR zF2SMM>2TQ7csTrUBJ|9bFljF;5$qBct`PJ;^f@=HAzme@5h&%smZ-=5N|6f(NykH8=x_+Vn4^B{YKEs?g!AT;z!_U6hb&Y7z3O>`w`CP z9YCLZ!-&3nMiKo|x)c3FdJ-;?+Jx(jcEo^SOEA#AjBxYO0)x5~67FuLV6ajO;bD>k zJYE-op$f;r(Co8d*wbxb_~9tvnRS#Hu^|AA^|Zu6RV;GykZSvbUrV-2akm;{n+VVP z(5^FX+|VAVQHQ;(1A=)FEJOQX8S5B!O*Gi+b0Wb2yK&Q6P@Ue{H}~Pq>cg(*(e?YT z(q%e(j|xicLQ0#1B4NX!bux}>(mK_fuUHn1)+`O-Ty%j;EN~+JSMzu5_y3dmI#C?d zI^yg4SEECn`MPDVAXsimgXMiiw#Y4n#-PXRHb2`yL=K;eM^>g0QBfuMu8bf&I&c9V zlX#M$rjEs9*S*B!1{C7)^XB6TCYsD{uQ(=AsT5Cqx1o-BdT0Fjf-pRpuZW+r^Tbc< zq~oW*oWakOCgSJPPci3D?_(~+OvNuotYuPnGR&o=bC}D1-!oSxj%2R7DB;%}ZJF!N z+nKb92l2GRM|g&u4W99ahu?VO&19Y!&14qG(^h zeru5~bL-$tS};187VM72bH;vTa(*64=i1GnbJqmnd6TX(`9n`I!ajcU%RFHpzgmLO zhAw;&!V~uI9}A|0{rj6|P{RIw)^=w5_gUMS?cZl@XSRQzwVgE#c|t#y?cZnpShjzk z^<&xoeb$d<`}db^;R*Zqryrq&{rhg4C}ID;i9aRm-&Yt%vHkm+2P{BKpE1Gim&RG< zO_?vz&^ts%J7Zix8~1f-pOuz&AMV&^a!~jcoImd+A*XJMgOfF?|*1^2fhXT>f|#zaZcZPQb(^B^U zsH!160i^qTA_>&U5=+TgUir^WvI(Td@>WJ7K=$*>QW^xXpFcj*@gp92JByCGmdWfo z1+eJwN<4-#ra8HC2#fnyZ#>@95KEuqz|`Q~YWg(SGndKh--F9#w=cu*nE%4uc}0+H zF8fo@6U?W=tMq5%T>SHyWTw148L!x}ovHjyz>D9aQUe3aD{#+?tOW^5^;b+UleUX?@WY(GoFX~&6H zi=)Wa1GW)u+$Y1f7RNwK*&q1%Jn{Ve2~AKjviyo-m|D{P-Fo*!fH4Ap2m+QMya8S0 zmS~XZD2xM4!37esT3x=`^4VOBuw}<~KvmrD1{-f1q@c*9$CXtier088wr(HM#h^ec zt$;!LSH7;!-}S|E0w@H(A{stN_!D~5j7=EdE7D^r6Z!A%c+|N~_^#9;49C|NjF7Jh zO{UIzvx!}C+3U@|`o|}R+At?=o=N!0XYXK4O2Kue(f%@P%j->MZaf)FXC9ixNbJS% zH;mYB&`5CtoX%+3c7u37%}#e`>*r@S!RD+oaAiCrLOdH-8;OYG`t^u>kH*W7z*Y#B zLJ+td;q8En$Dq9}zXUxV5jnL(+OGa@e~Pwv{VBT4X;w_|giln#oEL1|EVl0+*915> zHKCtf((ZPCNxSPi9apOs?;ODfeHE3Ecu`eR zmXjiMRFF85`m*G&nlks9Z2>TGk}hLRmT=EmEPl@Zt#(|F#q8_S&Oi@LLU0@cLV*T^ zKHz2qm7qN_0RcN7%{kKSDTvHpj-{U(fz?^B(1?Fbn^ z2;WSUgKs_F!ryy8h40Mr;d{Mn@B?JXG8<)B#^km*USIIqbdHFgeg3?TS8pqoM;oje zC+rb8B0##HK)Rj?74hnN1ozNmU!*hE{Ws{0^f_jdeF9#O>3^cWsFlz5ZT!9w8Xa8f zvC%zVVtr7TzGx^GS!4epzk`T|V3i6DR&^Hby$c8)fF|}XureZ#Fc$h8`CF2Cx8)b` zZhJby8a)egIArT+TN+)(vSH4xmmjOqx1U75w=oIks{RG7FzGatRaUHWM$iwzGDNRH zgxP*lhv+k*l`+8^Dygu+0aWJd%0tiJJWN00gWd4ec%B1+A@aoYeIezW%53j=6{i zdk`%iA^e)tsdA4HJVwAiZ@Ld(y?^6d`0#AaiN+#E)!msl}QgO$SZPs(E44yd|PLp zdi7>Rk358bZ9aNP_ib8wt-wULku3j6^!IYxzp?$jmUds)UnV+yfFP(h4T9M7L5C$8 zB=A@tGgEQBM11R3jAJv^Fd_P^w`Qc(xL94jF5u3EXU|TX>B^m*wx>>A<@jJ4bo#uw zN@w-mGnMR@;!O4r89=&38k4RwOvnMjPH^C0CDP5u2oCD(feRSPY{+x+tKEI+Yg|X|phUcm@imyPdb zwvmyZ4PK1a`vAgc&;>oWC))d^&ke2W3A=gv1*zK2iBT=>kEtsyWz+>f(i$(kaLwc) zm{#s=MtiFTrjt~T>`+at75Na^p|7zvcq-nu&l#-UkTbY$n`4+>kK2r1c{XhTw&Di& zrecO!N0`4b-~1=~!CFU1UGI8x#lg7{tU-LfhMoC2@SEF{|A5$R^_eB?nn{^mf6Nx(-qNbHo_~>AG=76o5V)bIlomf&1v7TXDP~B#|x@@gOl{d@>sM- zrKBvXFQIsi+7>lQX4I|E&>mm23Bf6}b}0z6{lSiP#iHVHLl{43a{cx2f$r7jXk4ig zi)u5%jNy9e|91T-s#TwAQ!D=A)N~J4+@q~CN-B-&)s<^#8iW`?5Q2=n5cVA8o>5*M z`LSz;MZ~?aW#VTn#nKPn!f#BnkjzKh_G>|UvZmcGw$DM89i26()_U6`drnDCMYx0# z{nnB^!D(3_w43$LN#}(&N!)W?+4@52=8b$rivb80A-XI_I2^D(?EAs@|5tt3$@wKr zE&4T+KWm=wl_s*t`jEYdCa(|{%?gxzi_mEFc=V2a7J|Cu7q#;wi1j@W_N-iir7wO6 zC3q?LEXdim85ho_@cx1eW!RKMP}&S2Ztg@f@#67r;$_ZuqPXpLP<-|{C`n8JuXk<( zrTO0hX4w+(Cb=(o``sk)`_>NNopT@Xex)<T%F+NSQHaYpEuI1_SR^Q zQjM(eYuL9?|H`Kg=(2iqDxsPPbRYFbUjbTSOHHUm7nUh141_$MKmi_ zJRh}6CWiG#|I-a&ckw>+*WLeXt@WrnpAmrRqH|Y7GgYh3XADC=dl>TB!zPJ8XT`X# z36j@;CyJ>|_N#UMbaeepglDnuzdOyFkpBP0KcesR^?UfE>z5$R^0Qo+&$D3Z*Y_0l zcZ+ny*Xx7O^`Qu}`dY5U=1PS0>jxFp@&C7=>wiIb2YOX0JWwe@U#Y}Z|K0*yF3!HO z`uWyMHElnKq8j%kp|`@~(Ptz7Jd8cqPHt0{Q`Oo?-k&HuAo=~yqR+U9@FnnHKBpi2 zrStj2G9d_8ph0+RATfhEyb}=4BX=faQd9^k#F&iT`GUv{*CcbN>4QH$E`pznCW7*U z39usF8C0Hjf?xI__KGlqa-o$#en|yX_|cG1oMu2O@yiKi*UzMi?L$Iky#uKRQwg<0 zN1-|sM`+Noq-Nnopp_a#YM+S(I{PUIcLae};j3ZmpL{`^Sw67sBoEMT=wPVpWDfMq zyFvZV?u3DwIy96wBMjemC5@h_5yrWyWcy36hz>_zk{x4j61>IjVW(9`h|c4_K$9P` zfT_b{*k#yZ(6!waXl6kIv&u)Lx%vWN{*`@oOx!^m<_|B$N`d0KIK>JSU#vxow78dI z#T|+*6e-d!?zXr)#bvSL?(XjHu*Y|oT=HEmm)ywvCX>uF&&*#lnRn(nd2Mm(__n<1 zm9pz_2WHU@U-sqcXn6f71jj?GWgRx%F4Y)fRC8?l4uF*x05;e2kn!Qbq5*2-OcF@fnjcc~<-h2?+v&0%E zB4f<3bF&b!eQ_)3++r}a@5G66r8t&%mqX$E$p0i!|DiOf^!!*A#qT;ArI|y;7QEjh z@t$xK{GDU>$&+fh!?Gk{8P5G!Pr64DCyju9;JZ7PzCcL&S2u{~cWmOv@k@oP^RvYI zH!g*SrE#0dfglyxGNoRA&|)naclOcDTOL-6ilvlkPLzqLV@o{~?AywawFIT8owQu+ z)f2T=BU>ubAnjX*qz;ZwmsvU{w4EpKkw-D{>qRsPepFE!t$FfQu&?m_HmM)+&YO9V z_*ICA?zAX7%dOVv=;sWAaQ4T+CB_)_O^#a5@;nLig_U&-o?US>5i)!1C#xs$Cu#>fRpn=CCyOu6@XBwhJ#ZxEsDAk}9M1qBVt(H$O|WOqc^ zY@E~#%64Dv%XXiIZv=;uW~(N0ik)RpY9I1aT*jATpB+9v4)v} zpO!YrSW01x@}me|D-^X}pEI}?5n-nP-%=Ief)>RLg~J!2kIpjWG~Vl4PV4kG9xK{) zdwi#x2TY7*0d0P~+y?Dh>;pz&YRf#oES-xDu$+F=^zQM6^IDNjxW6G&L1$)Cq4|6G zGl>u7uQ1T?I~AVNZexysb3QQ{QW!KM4ljHx>J0`THBxf%sKxZ6LWcOxU|w%W6F&@2 z$v&$t#R}T9s|iEBpjU6Z(}XI+*s3>E z(PR3B!aW%gIfE$L@lUdooBL?m)7OS-TWFr9Z%i!;ZR3X=7SHht-x^=piz?k0d3bm_ zG}km17H@H`nEZt`)i#5sgC5kE-0WM5VP1b0=Dybrbv}YU*6SN1zqLhXUyGf{vj&wUUsHXQ(}B0-(aq1b zl%!w%sY{mq(k;4d)Gn?$))yrfXWYHPPGGPK_`Sa2d!^M9GV0V;mIEv`0uhp-e`i%* zYj62&qek^_)#mADCydL~$92tna)TCB2APHFlIx>X2ZfpflAF7v)cfJYh`fv<#QL=c z;?ywIJT_K%YVtzH>MlpDgiYNJ`;(5Qd^dr>3HHNVeEU#w3CbUYysx6FvG1K(3KW$R z=Ptfv^cCAzG}sJit2gEkhh8l;s#88S6}9$%*`zd2`|M0srLg3#@^_6`LD`+S3L07} zeZHaJFBf>rhEe~I2hWAS4D;;hA80b}{H~Ah>UO!mBqsEJ3`5Is|#d_KEq1oQx*AXbUkz28X46}{oc z?8-jRdF@y{&jr!@^j>g@7pnP#DI(Mk!#&{kU3wx~Y^0ZHJq}n=<9j(vz4*$;UyT9#3AQyyfQ+DN8zf@oM@4{rT7EHWNnuc8A zRI8EttGv3w`aSyN`Heznw@{1THYIFj3Q+AK79d{u99_mIu4h=K9cP1Dc2xcGhm9vy zbM-A9Pmm<-^6j4R-c~6=We7Ne|kz$0>bU_TbnsV2C`8647K&CVmM3l_% zMPBmTIhhRV!--hy0woWj>G9f+>WH1~TZUJ<*1RaPlQ!`%Gh3VHOWOUvMs1=9~r&2h_Yt?i_Wk%or1sBh7V0euaBJzXLmM!1ZIN1QcIyv!7pVKEVB-1r!u9v>vP zUixxQ-Q@@60mcj8{QzG=_{fxP%$jfgzzcgJg=dc8@Jt$EU^G~a`0PdlE^?g^T0L@ri z(W&>5i1Oc1%r}ReL#vZ_qut?K47%T^`0nNMJQ^lS!i#)A`I24qqPfcqz*|ynaa}V% z=v!WV|J(?^3!Raf_NQ(SE!(Mv1~{LJR;$T~17nY1W|$FdY>JmfD2ppCs*r9hi%8LC zx$nN*IG}~Gqrei8fAl%Mu;y*U=<=Sb@qTp#oJsbW6^quUJg`ReEAQ9p9lQ*WP4vl4 zLvDeNbc0)~>f~;|L&O57%Yi`s>)!d z>y699j*M{OpCFM8&OAbRPL`?Y!c))^xs9jygCKkmN-z4z=V3L4QM}JYj1*71Ttrhw z%K{NVmJVjPZzR!FO^c!W;%@In`6Pd;IPOzDT+|<8`SeS3*esoCfcLEMpQPU*FKKk;Jp$+nJ3z@5n`)`TSVjqO3{&o6rK( z=09{~PXaOvd!`?<JlA|XhBp7Ww}E5C@>xTE=J;l{yb(BDOMf8?uEIN zo+KZi^n27~@hEWd;m1}#Y^p_#b&s--V2u)@D%0ZJBkbDX9gC3mA=5lBokp+AznAL6 zX?)Y8h>W5w4;-s5yh2z#6(x)Qb^e51k=T&-G?fBSBu?!X;WMSzyI5wdJbEC6%jNqg zX6iRJ(#?~eeAGwpKT>l0yn_$sz&#q9*{Nr{)ta+Z$su`(C3Jl~zy7e=Wd%b_C68F& zY90(dNFNEFi?>nLD<3VoOW+gOr!Ca(rg^~G6sWZ9mAviOw4Wk6s9da<7avxK4E8{S z7GNdwpF&3+Uf`#Kth?_=_F!7imNl(!nrXkOO9iM7#nWEw70O=hNHVhXtE8mlD5sIB zW(tVrCw!_I(8x|OjWpM3Qp?+GSy2@;D;x$lcES38Yr>R&uXEC~Ocp-2J^Ztm_wF@k z@Nt(ar?Xi_ed;@i4ZWyf>nZYQVt(hr+GBNi$NG-^nwAG%t7g({Az`-6sj4^p^l3@f zuC&FykhBbZBInJv27;|8mkm8W;=*c~KYDvQzVqqkmQS^7j)u>(X@4FW^XIg#RQ;?~ z;oqdWmr;i}Q9f&f3Wf%4n@5Mg-gpVQRynJvj9%;Q3foYm&!eV$PsQ*~F^Ad5VRD7p+N857IPV{fy*dn`pJ7P`0N8+q``^~?g z*r$!|`hG%KS7Q_`C`>&@TQJ)2Ekxai@8~0hha~Ii%9PjesWR`btWosln6zszbc4}r zGsDgDul`E)gX84fkzf-p1G95)ZFU-+%&0iz_e4Ac%Y{EwR?14p=!Y}UvWiw7*Vqy`6kVE7m)8F^}oysfnXvjj_2SnHIH@ zLn;Cl+2eUN#sfD|1!^r78j2JxUtGcLmlfWsqpL%W0iZ#*3;{}I+9Q9aP#+;>fh1u( z1{1sNFa{wSj5J|0cO!w>AU{h4zB9LoY~2Py0EeBxryO>@pQQu1*<;=4Bbpcp zY10cmB||Izv_5N*u%U;g1Em4*S2Yul6(G&fx7canH0--l1InX zDodI3?T5X!qmLZRAtk#a9ti=Ly%WTl>Be4Y~vAT*hj~`EXcc0te zhAxXnLy9RRMP~&PAbVh`AMf)}9=!@vRW?k53}$MG4U%eQ{*^?kJj;~!ZZowPWistt zrt+>TdXTX;pCovA{>cjzjW2Mcjo-OW!^p>DyB??BzWWU8vdb02dyHkR(vkP6I3a2w z3(tjxwK3Ai_tH8LJ2ac?T3K^-ki1(Oen}p4 z4q4Cg$lW`>9BOMNavd4fUozJjza#}rJMD7FiR>kD5btS2e14-o*N9HByt{J{0`1{G zsG^Jdp@__C;w0SY2ce}>x=J15MJmf_*;Ue1MnC=2C2`YL(&$H#)I_Z$2&acyc%auS@gf|;%H=rJ*{QL`RQvU@9MsC<_ri92W4AlCJ?-(CaaQp zfpxxB>*0yHYKwIne2W!e!vVTEyb6;vp~iy9FgY3JC4xvzc~e4J`I)i|U26#JFy_tKqzwFCB!6ZtC!2 zULl<6e0Zln%J$^;7>7Kfax4XkP>KE&!k-BLbg`qrz;5?3EoCxFh;Iwa-9v>m)>>OO zH-SjST*qrm)kO@mfcp5%9JYzIeKvHmJ^m#COK!Bu(kI5A+Y98=YwmCFE=8DFV_#}- zdWHVZyBNd8`nZ7#W#S@dw&yW39L-Qm^ZQfyzVDWpB}PMq2LwZa<>ip--}^oVO}+YG7& zEhVrB{EtTrSJ;MIcn%et=|(hEV?x!<{Ao5NcjCOD`GzhptsP6(&&R^{En%(4a6Io1 zrWmiGbwi@fmu1H(Ys=X%LV3=OD7)}YmG-pgjm8N2z23$ey+y0;+z@)0Z-+j9YiF2?xygxf>a$46>^1zoTa&()h!s!$?7L@R`sQ`QwCgq+ zWue~$16x+<8)hjh28$oOdf%C?8KFN63qMO>8+O3?{5s(UO`UL- zXOHv%5BGWPLx1b`ps;FrLo9#F{0Sy*G$?^9DK-egCijOVH4^4V2nZ&g%*|7~2hF`s zZu%5CT=+hqtO%#s{TJUl|6ITo^-R+2*daOP$QB#H_a89a2aj+}R^A@%h7Tf=F+Q(J-ZskW{nt9cEBe2&-Fq zmf~-oB1Q%U}~t93ZEM*;|fF7a30d!aZ)dZ*2!zilp8;RjfjwM_XZ$bw=fx* zq;7l%A$h`U+Js;q`Dbk*wvd1WnmesERKMSb64c>OP(l3cbOO;X9=ZPe5hn9XGbO>= zbme(I^C)()aTYUJ0uOs~mRhsfhrMp)or9|I&iW=VXYe7e11$?$!4(q$aYxn&11z<3 z6DM!?Zy;4??5yz4MMdPmWSh?zspvG%^xR-B5Bc!7cW}I%swHfrN^0lqGQ;ix=mqW0 zRzX|99gTEtjSG$Ad#kL4YMl>DndUoJ8+o0S-Oe#~-hp-V`joCs zL}izKe)ziEcfcS&mcrd*R^-nceN3s?Qg@liySbDXeLfCvU6`aPQ~HOdNE2{cduGOO zi3YDtb>)A6V}3rnrSwpze^qw8k;%U+_+dA&_F~>R=TG_en@5fhgH6|;vc-x;+PqT8 zC49)JhrFovl;X8ZOk1=9vdX%~6YS8f5RLVgQ!ivGCJOJ}SX8@hZoTm*%gN67Lh^Fqz|+Dhs-OQ<`@%iT2!i z$716$!^1B8y1;`Z4>CqO1fK5NK1(Cm9#31vlGDVTd~){Ue&BNGWqM<{WFIJ}OX6EL z0Pf5V7ws{iYzpk%*&Q~ps{PYzeb8qHq3$O$H-I%+c$Exu8)r?_r`8NJJ#|G{F(mCC^QBLTifE)F>g%|d{dp@VC z*=7f%BAB*?jMEHGdh_7sgjN#mlfeO(thu6CgST zJtiOK=kOvvrkC8(urWMYq#@K#ITy6nmRS?0m*i%IoinYY<8iLEP1(;~T#WdK6IySf zN8NXL2mLm?OQASiHoa(M5F-?LoP)%_Kx=ymq`U!^4qBE13a3mUsB0lMFfeDWWXarQ z)wa>T^WppKdt&9LMM&1$%3+bFECQB!i<+O%cV{b^ecP3zWn1=*M>xiA=Dqcz1*zKa zO9^WE<-?(v6Ps7J&h7X5zw1!2Tu$5Mges+QKMK?wJI$^Rk>1Fj7gY+Rf&`8_jI7Nj6ubUmly_&<+B5mZRfSOX*JW?6&qP;sIp(F99#{yoan#H3yw9;w zJ^G};mQpya5O-I=0=`d)NbSG(9nNcx*OzU8s^6cR5O*lAD}eJ%;`ui!EwDhQnr1MQ z@oq*3O$4IE6 zt6ME(1q{B0&&PWf>-B+W>jFIYj()=iN&kt$8c>M^+2fkaDFY3Y!efYPYdRV37EKvU zp{GT}P?NIbrhj&03Z^-%pey$22!g97qvgP+ttHPT6S3l^Ht!VAlt~oS+cr`ad4I?gK6p9s=i;7 zspf;bid;tI*j9tP{m-|cZ3F^R`_5A*J1*K%t0GpNuyrz`g$}+4Vbc5`C2EHi!e z6eo^vCK9^y(#S$I@b4Qyqg~KZoX3|^no$wpG15hMtYzjKL3yB)7f3@L`%+~W@;RA- zByp*&r?_ZKq*-Sd(w1)%v6YbH44F6f{jy7T*r2(MHV-N+`<&`lvfLT(H)lnYtBWy% zxnh|!I-ZiuvOBxfcVPzmXRc0qx?Z#cy;(Y5-r8b0W-@M z2Prkj%5@*__1WHCQT)A#r2UNPHoL5qiSr%W7c;*YI6M_V@BViri6OdJc zBOkwsivWH5Bji9gvy9)j^}>Y@xkiQ__~1=JuPY>x zDj?AU`iTdsMLKFfUFIU=lNmF;ShGx?nsId|1ooW%`i_z?Und-s2*%69q0sn^Iwj!6 z#QHWMUVDlMz_`BnLJ1{+ILWy;B#Pg$c1mZdUI2&`V0wxbRs2y2FmyqBE-)t~Q6@hh zWkMHZawRn*-U7Z94A%tIniN9x8?3Be6dt3Y& z(^08EZ^8o}OKJW+q!0XV&^(8#r^-N3zKECG1Ca zskFuc>i!;75r)SN6RflMctXNW|hhg4yPc5^RpBTz#~KXEEwE;^|ZI;w;9 zRTs&T1Mmv!L#cmp!uC}VIQaeU-yMNBWvkphvrcGzQL*?E>Ojz;@t0H(8Q>DcVId%G zF;t%?cm+ftfoD*%;|YhWTirChTYGSj7IIvru=d*7Q8zRq(yPc%T5%56TmgGj;B<){ zGXhxvuMQhXYtWz0?nT-_p>eGfMJGfuK=PHbZN=d)Z`NgyE^0d}YGcwsVbbI$9=D8m zaZkxNCJAa2)0Dd7<3VCtC7HNA)&WKhTI;J-sI9hPpI&d@2K6a<|2JubFV^2yK;SB2%vAC1&|zLnRSV>T1#FDsc%WZo~ zq+I${{*dxq!#S1Sk977L|ImtKF`owlZnja}Z$mHfuFlYgZkOJ;5RHHRJBtpq9AJ1| zo=1+|{=|9w>GV{S*+OPrf_JlsOHbS7%#91=D%cc?-K2}%#35b|g8!mDr8kOonjxZ- zjCbu@^g5DP?LV=|*^I){z2}e}WXPRokRRq=Gjx3cI(eOeA=Tj1Pbd|O2%NBLq{QOW zZYQAJ0q6_?g58i9NYMQBQGOzRI*_0VB115Il_Xvn`=vi(`8oMjBMkzr^DEkbau4+Z z0k(}ES7*l8mE_lZx|l*%QpGBg#cGm`S*YY}wEzt#!;EqPy03Tjst_#{P*Jk6Vo`i? zB8*x`fLt?$4i5B%1yK;zN?cz9E6}O0t$@H39{t~I+E>CP!Eo#y@|P9-gkEgecgRMs z^4il|YW<)!S8dD`4?+|dcs`6>pGSTEwIs~IBdk>~C3z=e1!yJarN^gABMyRU6h>;| z$JE%`E&sARS5v_{|hqkc_3#C8&i*GNHur+|3BF3-vhJkRL`Wep4t8v z><=RVaI>&Av2%0!Z?f3`Z?YKA^AC{-{v@^ejB1Ds;Qkltu@3 X|2IrD^#80D>T^~@dhTWVY=Hj(dk`44 literal 0 HcmV?d00001 diff --git a/tzdata/le.zip b/tzdata/le.zip new file mode 100644 index 0000000000000000000000000000000000000000..7e9390b15c8eed41502fa3404c0fe6299e258c45 GIT binary patch literal 91147 zcmafaQ*fq1|7B(}@x->RiEZ1qZ6|MR+qP{_Y}>Z^#@YG4-P)@E&F)3_Mb}f^r+&|= zuG6Rc$V-9#LID8*fdm0Hr<6y<&b|TR1_41R00Dso!2z)`aW>Gjvo&#|b2M>MQHBBm z1(!wF{ZDgohXMKV>l73O=9J?Fr{l16F_niO_uos#Of4pj!_sTn1e>+xSQnc0 zo@Y@og*wPYB4_^o9_N;|6N!64j|_nMer&=u4-49x6@s}l8!z+dE6jVZ@2i>V_dAmp z%GAq@`R5tF`peBj?Irzp*v|JW`S908?dSIwQ0^4;viG}x)gPb917W8E4Ho-Iv#$61 zQ*Z6mTklqrm9}0&Y z4USs5WHkc9uzHzPD#+6(m21%*jYKD7}UoOX?$&9oEdm>xd~n_KA_+05IK0_X9O?EAJ6E!b)`v z0QsA>wm+;o!xag+>NDzRniQCzWnVb9m-I!IEl^xzHY^|iQ z&xmbP*@0j-%JWDx>r<<~(IwaBUr+2O7)3#xA9t%j7c+U$JQo` z>uHEgGeWZsIGf0z7CNHc)}r#wr9s~tZ8N4ye-rD>%{eX+YwlCD=R;g1o(cz;i|fQO zae@H71e)mJ3??GqNp_2nMJAJ5^l^b9@B`9(xYt8N-4Ys}JkHmXtkv1RR<%ef zdO^kQU#x3^PC8VrjV<$cj7u9I1nW=A#~Q_4{k14xaWk0`NdB(av4B@|^DsT~jdsPysN0<+k2*2|+9K9IYi zn*rDXcWUGOH^mp0O#Ab{a{Xn5D|-UCkF<*SOzC%sviJ9|QO7|JkoQ&-rSr*u#?ep~?yhKVC{0gHWsTeTbA_}PCLL=D#Smm0&Y$FwzrIWi zA7b>=<0@8=x1IYZL`8di zs_2BYEgU+&r{HOuv{`}nlcLeN_OK$tMwd8eVpHGpjz4L}FVV3^sT3BpWcH$jj~KX? z%ee1`VQ{$<1;BlAwiCBZlu=0KRH|9BCa!!i_~b@S#G7|o^PBjk*gmGjT#_Htx} zhpbZK<~-i)hhJyHI;Eh_<-Q8JN2?X(zuC|wmZjrNP|s~(iRLLgls1~swf#^e&4!US z29=PLxX+M>Gw{rM3JRNkMFN(gUY5#~90KDLZ5&7~v-HAOElD>hI+2(_R=99{$_-YK zR8rYR@%`1}mvWWeJTJk$FvXmbGmIFrfIAMC7DEpPD5aJw1&?yZOIeR~h})}yEdKo5 zoyO{mf7zF142knpaokx>7w>ZO{C=IeH*DDTt=xP7g}-phf_W(xA5@|w^j`h9te)O6 z!|p{E-yf!_{fVZssmLpSH;fcdzhsC zvFN2hdXYhWo6kiA`WUP=B&mfz7z0C+rW~&RkaSROJ-@9P5Uq%5Y#BND%krvNrWoGa z$x9|01g$iICe)fOSsc7?hVIJmM2buNrvelYq)(YDXx@1j?s#>zRVU#+rJEouLDqZ5 zhzXsC-jW9*>QklU^NJ$gUJCn6Ld+pSo+*_(RiV0AHMr_Xygrq#_VOBxyr3{>Hp`E! z1ROeeeDOW+EYl$y(bD~%ln$0B-o7RixAy2rH=rMd(Y#*UYq4DBk1mCq%qIqq#<>;p zOfv=4F+Wjt1nC?@mQinG2};v3V$t(i`lD?z?GEGa_DX8Rz&i?iE2Qu?IjK0Yxw7%5 zaJ(b%^J~s@{HLGCy+RhW6@+ETp_=?9X6ayyAuqmu>ycR!EjOGav<-c*^Q)B^5GGop z)GM?U8@dmOeRLuU!YeY+lg1lxe(hsN`gv9I5WJ`J3)+vXZ&E$2*f?>VB24uo@)RAh z4sBI`pp5t%z@~Sd#|}d!G3M`( z$u>PL-5;oFDDB#0-M7&5Vs)kx+XqjmFo8tzWYy+09rTy|xcg)l|3a>-g-0UM!$=~d zY}#Myt05_>j9RmW;)NnnooJ8#)IAK)yS#)M)TLmyIn<>P%JtOKM3z9H zu-fErJ3=eIh1R+n?cv!|vDMV7P6!eYElJSQzUaRC#O7XtQPa;SYaRJ50j4ukMoGV&aH zj}>BLVm;2R*iGM$nc*|(1n;{AG5olvbCootpg9RttQns^7 z!>Ybgq`ZQThVDS0;B14n*XpFsI+EGCOA=-Mos0T#PrXgtxZ+Zqmrn%;YW$!gw~6D! z8I+slie73D`9cHwKCAJfeV^;~EUfRRaM1Rknm-m~kUDV49^OK^!{r6;_Tp4)PGxq& zNO^CVa}~7h&f$HWVqDChNozwYf5f(JmMR{-jZQrK_dP|&eV`JqN&y5O?QY#03n=p~u>(F-__rTQ_>ci@=HaLvkE zug=B}>$gkC=w>7!uFdmU>)k3e(?Q-1o6VbU_jNn-ZMP+gCA9W zqj7yq9O}2aB>fIca^2G*5#8%FQgE>bd0;@|57U&N`oxyg|xTgqgn)azWcw`b4Ze+(+m}VA%+Y zIeiBAmO7&fTdC+_RDy%1+=D@WksIrWa&o^Z`@f$(^+NQ(RLO(I@;+q0#stuY_RFUr z8WrW3s7N7eKUGz9O9p-gfF?np_uh4*ZBY2%F1gKt4Q&73)NX^9fA%JG{Amq2{Ipt> zhdw1Bn5Lq#i)7u%gMy|onEvzCMH3w@9?vgA3iF1tuX6CN*CVHg!Z0w^b=S|(Q{xPo>2Yn zFz)=LUBD)PTP5izG%BF{wez!T*Pqfdo!EnsD70GoxUn`*o~Hj(B|_v~zHyI8EgfWb zX>|LT=TG8tC`I%^7n4cvm&jeDZ=Pl1T{_q|KNU&Amxb^^VKz+r`@X}P)Gw-xx+F5V znD>{?ig9692*kpM{(H7(xq#IfJ%I$9>Ef7@{E(keAkzZt(f5mE4q;_Vw1LjXh z4+t>DrfC0(gxb|~-tOiyE8+qf?LHWK5hCU1trdlVfex?2@0k(fX)-BS9)n1&7I%{g zwwCeLg)p9OhT#(HKE{t2^=W40APm)dw_Jv|B9ftIL3mr&iTCG+Z8El{&dR9~sb zDha(t+{8tePexbx!&B4j1v?4v6pWdD z)OF0MUUEPXBFDQcT6EqCM!B!4w5rs(m6?S(Pk*!lL2?}Dxdy-OR+mvjZ-}fwM}G23 z+CYts0INOs&=y5*kopey+zaZIolX6G%Ev;irH#daWNp1C=vIvB#XJ5Vu`6!N>ay7{ z=X;!9tyUu06ldlJonEq=^{)tSZ7%xcCQZC5rrRLG3rw$)((1(opP-$HKUPN9G%edn z#-XvM_(IXXmbT@0LhIbi;%VrDy1qR+N zH?Ep@r~DZ>WSGSzw02-TTwcPAajj1)^89Dt&G$Y*ji8qnsy)?>|amo zsZT^QUzd>+_Om*hr)R%P@Rl378a%+MxuUk`w0|zER~4eVh3Dp2(D|Q4M?TM{g8JZe;;`3=kVQY;KPXc;O?WomKM{<-DS{Rns252=tU@IGhA+Sn%}%m zm&|%$E?mN&xy?a@tE(8<*-H!<#$ErG?W8L;J5K7}H`e89AOpu8p8snoTjiF)#bUz6 z*9W`K;XyQnf431&9+Bt*Mer5XLFOvoG~GKFa}JzD@cp}hI%FTl7veIcRPEY=7F4?l zWya%C#C^j=vi@-1R?M^U17@E=maCI@!cE4joIP@Vd1@ugT$d8~E1Nrq(&==YQqs3j zX^YYLZ4x)zHPsrs6S=%-E5}lCTh%Q7>7#@1VJ&D?E#6Jm20Q-d1#AlTO5Q`Daj@2b z0mG7UyyHILIM|SI`rb`gCDb)N)sif!e7%}^Nt>f)5#1HEG_i(b!MF*zcmZd#{mN!_ zF6<iy+yr=ZeVK)+WNX>b$-A&Rnp9H~nekqi@@$+#s*&H%k#o&=E$i&UC@0Ec z&o2$b2~|TeU~Otm0rMWUYt~s^i?tq^;U4gt#hYi=%ekyRd8V_`v&@4m7c=YU0M3>l zgya6)x{9hRn_ZdcUw@xgPtP;jaQsQS#CGO79x=Y|& zAlEg{^(XcIXio9F)8QierbjdlvG%O|Ce23sj$ zt-Xq4#t(PM$;}YP%_SNCAq$p0+L-QC_1c&j{n`8AdHr=7pDQ;R{kdmHf-Eb(v?EsD z|2KyaB8(dPai`1O9Qw;xj+R&Jn^Ow|-S_lJmXhvEhHcpc%iDARTh+CA=W|016kBh; zt;8K?+xr5?z?pp{_masV>XHO310l9k!(mc-jWz&l9)3o9!Yo^Bz+!raVgY{IYyjGQ z53m%s@~Gjnz9Fx7m;Er6_5A(}f3JynaD;9drP4N|z1~XN%tv?1$eGISu*fkW(QuD6 z(K9=aWtgC}g{@)8WbY+lrrl-T?IP5D9$Dq%d5u2i;jWAP=mwL#*xgN+;(N3%9s zp;}KnQcqcuBX2~2al@R%jOJn6MZ8}@RcN1nFd z=R%zzxR2t>w0~LD`D#=*lI^rSt|4={{u1YJ7^2O)(S#ytgmT49_CGn8PSic+J?BKQ zj=KgrH$pz__5J2B<1*zTUe|VkT9>X(er-R=gUh;6McNv-0ya9P@3LC%PKEv?6TI%j}1NWS_tO09BKki#f z0YBPa1JF{NI&mo3p5QggE^wt9p*9A0y5*n(UDD8R?T)wsthrF39+Qr>wZb@fd(V}k z=jGh!So;r28RanuUSi@p!7=%8qD`zE@Qu;D^TkiWB0)#kR?LpeB3y30SkzmemWz(V z1FA()Q^l8FW~Lh<8g}S{Vcv=?z{{cZh^wWt_oR_sI`~M!>rIg3!XNX|K68>?RckZLh3LWf! zlC3hXKre_9HUunI&SOO=Qi#v?vAOd7dev@m)UWZIZ!g}==32A0zj4>U2&c75Q@#?R zCY}isH7cV6Y0&Chc$Z=G0pKqkveT`d4#zVnX6PF0xO4ZdVJAkQ%aaG!t!~|iE~J-f zPe$7-A>`tpfN$C7owF=fhMTcZd(GKkX2#EGLvE~nMybh{-w5VzM&NZ?lPE`sBb)E z%X3apZIuXp)@t&srv&p@Au73Rcd^VBAfcTa;=qkg;KrgYuk1)}rZ39DfGDuQ2{Q$> zm^)JwD5gjsB-jQYV==9ZWn2VnI7$|_Gb`>sYhe-{-KLsGO9$U@bvq^)3$$cIV_Plq zLvE2F5K#&Y;yRDETtB9oLg0GVTGw-TDqxB7VcdjmR%DR%?nMG)SXk8T%>LP; zQP|FYkJ)NHccn0m9R5RAfbDoDK+)rf*^fHthgCAa`!r-1ZKJ9R%?!pdSIYC{UerZ{ z*crk6x_Ig&hCx>SEQcD|88(}+B@T|vGDW&+Kbc9A1Quj|YLY7o%RZ|V!c@xh5QPJ+ za6h7)3wv621O?c-og@D$lVg@wAon;4L@N#FXyKnQ=9q%W3))TUD|5;b7xJLlNE(;Y zosy!{zXbCM$NSK@7W>cW=`ko4&b#Jwy9fxfSL7@RhDj2dI1W-yQK@YK181-nHZ4b! zf%{n9kdSNG%3)2!Lh()*Ufbw}qYpiJ|NaD|M~)Q7vE(NpbKbqBsy&E!o|JhzuE8;2 zvWuCki?F)BKFM@CDc25Im`(ajWG297$alVB4sIgVgaTWgvRix!e0 z(=;RuA4%}#-^?0ZV&w=^Sly?LBa(J)zFJikr4ya23YN+W+7o_qlw6Hx`$CR!RB+5hMbUF z6w~bjx--SqDq%idP!)xR341IO!jomCXy-?B%j+Q zB6>#)SJQtiMB3lyEn#hx`Z(%sW?>{NlL1)3`uFl z@ScJy$lB;x7cSF(A9LABMw2rO;rBvO6#Hf;k0)fjKv^ZpNrl2J;eotb^!;@RlUz=b z&I}7BQ@(U)w-U13RM0^+^$}`a78n=pTF7V<-?6sZFzt-1Xqq&&T)N^0?>_gt^Ta?4 z9|Q5ky}e-AZ&h973Iq$-JnKV&az4m@}N5A>}*cL{R|8eg2Bt$1*wR(Pm$y_WTN75qn||A7$VEs;}PpnM77kI z!f$c+b%ej*yii_^tCl#2kjx^TUVX4D?Jhrml(H$^8B6t0GKHOZ*r?*7R*s*aoP1lV zpjBq#?x{F;Fxji#u>_P3a&n^S-!S%5Nvv|z+s49LKB;Og(Q}5WRvGIzLr&uo*OQlN zsg0YlTlUzBX**=(ZM>r0@g+NmqNNw|hB{o6v&De!ZMr0gS*RX+uMH=_aoNnh^)UG~ ziL)QR`73aLUJX2m={#L%Z*rKtcMo*MXt{yUSn7K*{Zo@QR}?v#rWGq0At`@trC`nF z&W|z`s0Jk-H+V0qG})k6?>V`{P4Nl?C&&i)GL?88c~BdYqZ@Gje`JsmsGNMAvnebx_sn_FvOQLYp_A zJHx7a*1ZXQUV)nq(D1@0K06#M_-PBhi~L^bqjAmN8Dj%a8*$kKY+0!sPP9UEzFhWQ zSC*lgKD@|;;yv1{7E%n*)LDj+X_k7vX@u5fGPvTg$N6+4OY6a2mL& z8Nq$o{3-s;2HEbNE?&dgQ?P_=7vk^ix?K3)K}XK|GSs)_mDV`BNEQQJEpJ(o9@^wH zsiGb#U7ACq4ZmGAcQX4fWRFYqd)8MnQIMRF0LqbbP^>sYbnB|f)vbVG7wixH?~dDADIsDgHX<`a!Vy6Xis z9>-0|Hw%x_sHP6hVI8C(*_IuwcJwe|i)f)V-j)er9M!rEYkJ6zb*(bGTAIeDG170& zXEpQGsqJI4aUYO%FSP|=Q!5Yr)C(N!;i)b~Wwx!AUi*FAv&4d|MbvU}jJ8xMqC$VU z8lpLIvJJz3un`aXb z&{l?+@z7|0^uT(GB|ZG@h{ln*e|HE_K%}B6UsJk`a_a>u*zpGY(VUtYgL1@L6QL(g zj2<0UcDX?>d%7)^IO12&)I73XPI*z{|M5}KpFg=712s1F{&Vu#^>f`b;j-i;(rMVFyl7;SaR=QHQ~15Hv1S@HR>|%vhFhNvgoquG6U!X zi~*J(D=+ge!|djlG}_7-HQb!!Dz6?#haFSFrOLMS+^povz|Qdz>K)1~EjDd89dEON zju7l~K@n^Pyqkh+&d6O2O|Tin4=>R9MGgIL8v?Od0n|qqZGCqqr)?}L3dvcZYM;~Ups(syBJYz!a*-ZTJF;04I_MoA2XHmvUo`g z7vdu}kgtCfIpQE@+L!9|T>o5j(hNMNahUdK7{5&+Fdt4xH-I}Syw)s@Mb5!^v*jLz8O;?adVm@ zJ+^;NWZ7`^QEjF^3A+3)*R=a$GgIiJuSikd(rv6g=AEhe5CPM_$UcU4Dqb^#%NRJe zbehz*Yf-MQ(#+9`^DaM$DX4T*)DTN6J@E^wVLCDPRf6!{vYDpP4C1`a0{%-!}q>s(qq~Av%|Q<0?fqE$W6;l*GtpOgqXCO8ad}J_hzTx)46G%dRNw+^oj@8 zW%w?8K-WGf7r9%ut4O`h+^`^HYM%v ztYCo5ouy|Z3tGlbg2k@xa?CuWlWR1g*5~Ge&K$jbz~yryYW=R>1^Mzu7KF=dF$#OV zDY2|RYO~DaeC;m0#@$o|V_JO=>RoO`Cf;zfq z?s>%;xi8r24zVLEe0oyuf1m}}jyO!*dXP1mAKrBX;>LK^=#%V(@ZVVkzcEHu!;P$m zOVK^lQT$G(%m-32SWl=C4*)n%_JeiU3|V2_F6OTnciz9syKw4Rj@A-5VZVJq2>(eE ziK~%qGC=QI)4?Kh&^Ni{Nj&&QZ_S_?mRLKcLC;u&_igQ33yZh^a_6r-uDJE9mt&BA zZdH>9cUH+^qje0I6*HrdrPRM(iG-*kdQvxz8$i3McjuaMfSi&@ci!x?PZ8wHb+1z+5j*W%~$>B z6};3mjazT%jWyQlOp10$C#r=xY>vkCTkURJqH|7Xz<$~u8|BAuzQU=`>q?5I-U!Y! zB)$L$f4j)w)K8EaHi=1~J&3y(JAu2K&x}34?_{cw$9h7hl8*^TQ!S0%b9&Nu+Kph+ zRxs&GNS?RN>Bpa@8F)T~kY~_6-(-PzJ(HV%eK#Y9S=_!f+8`iPkpH_x-UP#5Eo4W; zXFK}mH3R3d(<3k{AXl0H{q-_!Chy*wK=cm720oqvfA=2ylH4-QxES$aOUmG@_I!Ka zEHeMo`i_{_oNwQ20OE=bFvSa=diI%n*C`h%qBYz)41dM)CG|1eGf2N1B6RD+{&<7X z^zW39AM*Tl8%s3%r6BxuoeA008$Nmf-O+O3wo(1{U5(((=p%Q+JyM4AEo5Lie*%(R zU#g!c{a5%bw$$%)OQ_iu=aVP#xtbY-_jn=}1mI%~^~tMTH@FgV)iUaVc*;lr)7=xY zEAmwp1#~iEKed0>hmxBd`ijHc3m?-H&w353C}CZ>4CK_{<-8on^<=qFCluOvjj6-z z4P(nE!WZ=^`PHDV`_&*$0<<~y=I7sUw;$$#I5}`2TmO85naBFOkI_bn2Fed6=SjT# zsBdJxCce8yht*ayk-d$^xLk_Xa+M{rPuvGwA(xKgx3t-CLi1bqLD0%#VK6EYA!6%$sTi zPj^ztHxJSm8-(>8@Rfp{8kZN^gQ0L3ralkWD+W)E5jbFib^{xOB$W>mNEBEhK&~F=Yc~H`1QX$ zKM)IpdJMr@v<Y1*u+KwT$$_7ifo>-B{Mg2~)|&vTNRrQEfdpY3botf5#g%0f+}}SY z!}`qI)B8~saqKXIi*LV}GL&TVc6RKB0qx3kJDKfsT7|Pcs7}EoLk}oNURAr4ixK7_D=Bb+ zW-7@MfD;=m#&^B_TOk)d4v3d$v z$Jo7sT$Tr2Te2hcu$y@8)xXzU^KE%hu3aO1gYMmu7y>?Yq&6n%x~qgopY6xRz&?GT z%lY5%gX>RMThso+9Bd(ny&fxWN`X2}6eyUBIrL~r+8n7`?f{j#gpzh9qVpBCR`Sr} z|P*VpdkR{TMx`^`GxO=^D}9d-HLISN&SL zbN}#)My;XF+CF6*8d2Q@C(V#IqNix9{k*!6)GcKv(x@fYnR?8md^A(VTeX=?%>w{3 z!EPCHVa*b{o9yt5Khs0J&CL}0t&1S<yTYG zCdD}L1-B=Tdod+19j{NBW8ci%#gtr~jLofH7uFw*DUM#{((uSbUx5}AwOr=;xV!Vb zcL$!V``YREn$3SfE^VBsT<_qs*7fq$8h+grKH_TK2z?H`Tf<%KIY55-_X^c?6KB5o z;)7u$++59l!fQz1&u{N_80Sf|?Z0X{)W8El<9X*)T#GdT~OCr&*~GY=o%-)3ZqW0M?J(LmppulF{O?->_tVB5g+#w#Pc z-r2%Q&Em^@_xIOJLg~(yF2ZJ|*1(*(MMSOSlZ=uRlk6zWsF{)@lcA2q zTCRrSlXluyDEJh5%wz1|H#3vDlDv{blZJ*3kJyF*PU-gz_dV|^4VoO>_PV=zKU;qd zd$@w3LFB+oyO4N_dcCcpR_Nq|C(AAACZC;^-Uq=d=&-7#E2$tWGbul)v{%SffL0J! zU_D5vq*rto#~yW^w^Ya$E2DdnX*HcY<%qj9t(V5AT;OKX0He8RahG@&3pZJEzRo3V zi!FlQRGw&D5*~D1lHBS&%RQqzM`)~v0Lmx2NZo4Nqbh>1HiWMjk18^a-uOfa&GegQ}b~4CIK#IesX6K_(ws#M=_!t zF~$(1U%LNZN%yf!|0R(Y0535jGc}?+CWL{J52!2~v{Bx}!`sC#6A@+-NkBwf-3x+{ znf*-0ijR7(MN#i2lDuF1PM#iEb(u|Km9+svTpOBGV)3H zXTh&#e`L?1^31|`&*6t-M3MX!ffhAn))mDXHX_m;Q10@_bcb=eWqrx17`3iZUtXm- z15^~iSg<IK&~K5@#-YvgYYMyR$KrVT;ia$)%_GkvS(HCU z;V6ktm{bXRDtJnGiY)6ECNsK9J==$F4}oIV#xG0aOI;Mn6+O*^mlp*0N|sAj#9J2k z(SCSA<%44~NlqzNF49G(bZEW+FY_79{Ryr|=32~SWluxQ=G0=aXZyb&q>F^&qEOaG zC^D!{WD3vHNJeCmO~p$USh5Ao4Cojc^kVdcQ}xrcyhwFON2*FL>JT&KA!!dbPU$WO z2yup>k_x1sb!4gSp;itzlOIzqD8JAef#`@6*8{+kR5)DZJsyRIQUmTLaTja8EzZ&3*TyzS-MmI?Nf3q zq%#B#It@ zH{FFv@G6L zzr4x*!6ZEBF80=X)dT%BCki3Jq;%Y*$cSLp$ffhj5GS8d54rTl$`0pn7dvh+{seT0 zPqzor4M=JmKt)(0(D@%b!yoV~iwPM`!yfoT z;}?H-Zn`PpXN-NSW5v2U#{60rnVu6`*E)Wi|KaF`Ad~QFFprXsEg&}WFDa3sZHJQ` zHrBuzI33rn;Y`PPl=j+SN;s9m!r{!}Xd;tSr6IF2wknFbAq;c$%-SJ!43vNuhnIkt zij|BkCNCm&tWne?GdgOheRw3Q8GUil(2}qlyQ0#heSkNq3gJzuC!hDU2u!M-|H}2U z`fgTMjzqv@6DKy=7OxIVagh5r1ZYaH3NOAhCwGEsimbYsTmSZ2&wLgl3K`XIa86?u z;gpQQlo`>-AsgQ`K2x>`an5iqximogXVo*E0n+tIjMkXNhEt@}B%O{NstYRXWrx`$ zABr;%wZWRiZEy0t4qreY|3sL0hc_vEifWR5N^1TOBIu^7O4>=Ui1_$uH`@Qlc!CCdy- zCnQ;&)T#2|=mCGr4}eJ@hb9?Wv~k$Mb&7|{MR39=9J~KZ(xgWEExCz%``fK0K)0Vo zc|T=RP2xtf^2ih3JUWW!9~ZVS$_rq<%!&ETEb5qxq5&m&|ERky`lT!Xj|S`N7TVaa z0P&46^?;_trpTturqD+X3p)!v3%)ZDivs63=P+i83>sWAnfRE5n3NL95%GnhgQBy0 zb+Y*}#uUPAGAE`}ns%Ol(63&d&)}a zd(VT;3pq|2Xwn#w&MTf_TBJWxTX1}G=LtIrI4M1d7eYt;zW+4@;Q}N18w-`0^fWsU z7`Xfk_16_V6;ub7GyQ2QLH2TqP5D(BZke}0W=(c^cCAjiPVH7XPXSNmr>>XYd!=e= z<7vhj=lR=(qVrh?W=C*G?`g{^-1+or&(ddE<&s4JZQoQN7X&;^4&H~4z0Ym^Ze?F= z0O*rwx7?0KZW(@lZeDKX%v|l9-5mas-r~*z-yB4!aOhyLGb|>?z2fL&98l0v=w8S* zH0yUe(tT@ONZhQvnK{6GX0~Y7A-_?uQAjNmEd&pa2i=pniB%cd6xIrFnW0K=b+oSB z0A^r06!$k5x+^`+pG~Z7?k*$u@7uk-gMOv~wIDml9V{R24?}mCm)bqgz%2x;Q(O1# zKf`ebql^IkRr6IdcAGZqHZ$`-!ou!EIpL?M3?xJ>hO)x8gV$k+h^UAvh`pkx#NP4Q zL=Yk|;Q%H_hr{*J8De;moam}307`jygK%@qIVKAY6|JS};(l3I8LfhzaI)A|xF=FE zizxO%)K*krRQWJll*aJUFyKJfsQj+v&a7}6#U_I7iIFsOIAfccmbLoN=aRDix)G-- zr&WjL+nll$^q-3uar-rceHoHca3+qXNYHz(6PDsxwZIHan~rvB90QS1{`Xf3)~G{ zF`N?|HXIf>6KCTl`VMdBo74O`qry@#`l#vXZH#WVH}}W()66;e!erUNg|jF(SC_B3 z*8ry~rh(TQr}77*6SL{9X{BkqY_}{I*=Xu<)PuLc%2evLg73nR7s3%|-m zmm!s-3OBG$+-tP!Uv9oYd7wJ*H_+?)=$ab{4papy0KtGZZq-_644#JEGwz;VAs)jX zrXFbSjy$;@aPG2RHJ;a==RBUZQyY_+yWZ390d8GxC2sy~Gg_xSr#uJUm)$4T4;9|I z?x^Xx`~RG9#1;bxR|ELx1A?Bvh&TfhN&COYZ&Anp39(yRx&x{nAyV6YxC(ppC3nBj zG67n=LO413Sa1qJ#3X+i;xF7sND=meT|Z=AW)Z0-L8~SK2xn2Y$63<`+F2M2~k#i5?WG2XinxS2U41mjr6#Hdu{}+m_ zD@*{UNf5h9>?da-bw?q&XFhz6knit7{eNH-{~*yD{vI~uCfil%3ZZnzwcC~H3SLVK z1=fK?#zhyufKb;%d#Fb@tB1m_hRxJrYTa_Q7;?E9vOOE}!R`H#l#gIykF0foLv0I_ zY74Mxi;?{B+jzmgKEN*%J^?r})EGtxO4MJ9BE#SFaWUcUu(d{5ruwX3L9Gn%H-?00gxFGX zfhOwy+Ex7{>Vdt6JiJ3@t5LKR{jUG&b2%aa@F!*uIbaW0VvnYGK+`+G>FEOlL{hhf zT$ByUZ1&?S@6lD=H%gh3-c?9lt5#h=80hx=!byPF$*X)@zd-h#I~IJ z#hgZ5pN8w62Kr$5^||AH2j*jhqu(L@M@SRmCjP-xiuY$@!1nFm7!^i4Hu}HdVv$0FhysY?yJbu3Qz}oqOKbgftzy8zF~3a?l6oS<_o7~jyzi}4sX z=!_cnm^}Z5N}9$`UJ9tH_M)CtVw6M;O_DjNszn@3h9q2W*ztb)(C3HG$pBsicCY+2 zkkCTxy<5`^^J@`9()0<>nj@Pr_R=fj#e{vI`Pap3qJ;Fz#djtSnAEv+J$8A*Bza2H zw-er`ns3pF$`V)Wo8a@|ECz7+PNLaPLd!jY4$ZGCd^D#-1t2uizRD^Ci^yH$JDRvf zxP*yidO9Z&@Y5LrJ)H?#Oqi5$zzW8&yx>g=YSGJ8LsGrF!>#017{``LOe-_=`@+S9 zOAIubjgjVf&p*+4MG=phkD|ionHUkJPn{DnG>7rd$co@bm2ukX-7nB_pD6d^f{9)r zeBp)i5%TUn^`g+`qz=Rd9@_)u`6H=NZsP$ z(@X062Qw=#n`qb&Qz!cuLvP{K)6gZFUXnBZRTaw4bZ%V1{fZHc^&Ogo2n|G} z3oq0t8OtP+E3zhwIrkcQi|o+oln-W3BcF)G54URXs{YWc zx_9;7RbA_O)^ByN`t(K93gyczVYx_9Oj55rc|#`2B$W2b8jMFSbLQEXUaxGIO$eAcp2CCwp8zb~ful1 zqi36*5FZZ(e{Pow1oiBy2v;Zh)s}(^;x^(yJ|__jX-(5Q|~;8&mu5TSv}C za-P>RY|944Q(8v^bpxrj3@=62zo<#;Kr}8gP{n6TdFjcz0x>A7OqSzv_9m8266MSZ z*BUY23@SnNHAJ8}TzZs+tOBh2kYivM*i3UBwSlhhPV5U8Um~O_mD%OIe5Pylz_*ni2$(!?Q+kI{dYALWkxlyr&M-e96C0adoH!zn5uxNQe{0a5 zwRT4eE`zXy!a^y{bg?GS;%R@iq*c7M__h-O@t?##bAsnXC58MoE6CF&(~2jnp!)FL`Iag^l!37PVz71a>Rx86un+77|SGm-TDXY z^UJRe)4T1n+6I0=akZ?#F9km5KhJ)xA`H{_F#$y6fD|S5F0a6Pq2^AF*|+`#rW@jo zEm~dWyJ$f^VMnoC&COd7$e445K&|d%BH~B~6DNm+1S#@1Vb5$JP-=M1y-QnJKVpcQ zb(X;uMj^v8=Ricj2f0M7TwuaU5>{wYirr}Fb$ia;EPBS4h<=$EVc?&P>EQqdX4av5 zzOKLut#Lu71dBS8f+>9=d;{#WES2zZ_`uTzH9|T^(s zy?^LURmD!YwZ{A2lia0QY-aRy;%-CO$5SZ_K1|*j&9OHv@1}JKZZ%NqW@wvEQPF{d z;U;W|8|E9a2g+nRRO!t*$^khh#r0-|VDWMqHoy(C;!`#R#G!maGOMa7#Z#W$Uowy7 zatrQ4W4Y=pRdWG4FmBtsN0FWEpw)XJ$>7_FFs3>y7C!}4v@-s~?b6!LLi(h>1C)5v z$gza$f~l&RRlN0ody$f1WYVtpB&Quz9*G6VYl-}lpMDZLX14`?J#U;xjV5lB4Qaw* zc{e)}@b&xt_CxGT6^yH()6rblBv#LGUBmLXE^~g8kp8SWrg3g8-9|#Ep{n#^)}gG5 zxwKq)BTom8fx}MY+IwYwMQt^?3@6UIU~1=WQPgG&on#Rqj1uG)oPwn5sPU=Yi(J+w_vX~SsE>2nZS9EixNo5GY}kCM)!doq8NUM#e2 z-xk@1>Qb-G_&T)-sMXog?R|jDPw;L>#%v^6+tZb1;}@o#?2)HSQJ#YoAa})6t5^v3t84m9`N-@dF-Vrmv4pS_QY)SE9!Fhz2cM z3lodjuj5q~SGDWcaDXf!20?fcxp@5|0<+tmD0etj(>)=s#r87zJ{R{`Y0Hpi`s{`P zwyvUwHs$Ls*6xeIh4xlGs2#Pu3HmdZSB9+F+WFUBhQ%Vpc+4!jb>&i;^`th)9hg>Z zmmtVo%jKY%a&249bu9*x+BGK+Fv*F9tcFMp;>l7O42;*-n9z%CY<{-V!&T^3ofiS6 zhyZ*PD1mYGmFwsSEH-%IGX2uiMN+W`rdx{(dq$X1zM1v&oq9?32wmiE%&rB`H4bxG zGI7aoT{b2v>4VHxsqMDOxxe~jkM6BQ(;Jg`8MJ37931fSf+-?7{dcsX18ighw_Et) zqjb;YHD^W9_-dKYJ9G>(sPD_rLe=BouZ(p+QH}YdW(PZU49_Cstp{>$3(=IKxa~!&F|Ym+R>PJ7*`aH^xunEmNB2iv-J_zo!W3*BK~0V@vm4scv7LJY4;{ z`dUSnaxMuUR6`yUrO}FjMDL4GTDS6udlZ z*~fRpAYWK55KOH{YyOvj=__}H1Z%n`D4sf%L-x`f`hhyFPxhiM<{NXgx$E;xgeH@6 zc9im~HxxM4!kNTl27Q~n@f~A}XS2Hzxz_SKwPPZlQZ+taTd|cw4mV{D3 zy>l8DiT8;6J_A?Uf0Q$m1++oQ4{J)K!~8+eh0B|1h&CKWHhB~$F^S=f5Qois7M@8M zCu58rh6^1WDoCrdOVseO43k(b7Zg+8;QVPq2EJZboJ~2kM2fsr9@K@yEA8`0%fS6H zfN8E)BGBK;W7j=6Ly_crXEeyUL&#uobzEEE}fH3Y0faCK(B+i_$FVfu1zUtBz#+TWHnIC2#+~y z2vJ3K&>GYlG$CNT!Y3ts|xWuYJ)w1^(?{tHWSDOP%O5@n;kq_4@KMv zmB=MZa`wJ26OCdEF-|T~4Zuh@pJp;VR0zu@WC_oAjN`i(cLB7TmopVt$P<3opEeIQ z-_aMhF-fCFmvFK?C|#MeJ1QM(Hp*%?en=Mew6Ii5 zfKEY>$8J2#Bno8bzEZM#tS8p_NA`j7ypofON3G-a(qCeh{xi@Kz&9Ji47W?>+2O~K z!sSy|_k6U2D9<*&zTZ(uaVm0^j7#=G8YaiMNRsWljPl{o&6uuE`A3JO^^yGXk1I>6j&M#5)+E95 zh#}68bw*(|NzhjEpx`eQW$7%NInJ@!aL^h|rns5}z(Fp6^X?=NDCl7QuzRjKJsNp* zG<4$9EuB%i4`pIRp`BDuRHpEi^!98Ca{ zS~pT0t(J~HX6mhjfd^EfxfcoG1k6pH=3&1(xT%||M5ZiyNuuHrQ(;uKSB&XJ6I2jnw%@v%FF$!C6qZ< zqZ*(7ua_~?pMCs>XEDS-c7JK<9qp&c%wfNltR)%I`i4Em(+#B)eDCQEvr7YY!fFjp zNq6)oryqE|G{DR+`Q%R*jnD_a$N63R!>yNPDbSFzlb5A&i>A_NAQRY~xFsKZWaDKV zMX`%xBpIC$|L$lA9BlG61!B0m5?!%=428PjD^BZ3SZg(h`;ac#VqAWtgnC6zp(u6? zg}Zgf*z|#y(in_6#9pxk?;LU)016_O#L}RXPp{+5uH1tM7DFqjeGwh%wo*bVhR^+x zP`Y>Ox40+S`?Enf53aYqdXtEtF$3+s@OM4tX2i>dGb}+(aN~aL(JYF~ELyQDeY+4U z@L}n}xO4vxf`c0WcdtowU{vtqpXL%1FV9Wxhpe2(o*Rr^%{O)75FDO|ZCya~kK62@ zH6|a??*~qO(T{D0yqCF7LCOe5GsAxgTr=G|C){-0c#_GMQNdTIcGHX$ckKF7qa6r# zBJBe}{tS{bpwvoYZUu6)4Zm~Z=hhl_a%Fo&T7Sj24XU+}j+e-!$XG{%wK`MQUBbl8 zm_X>J@L{yi%=hg}JU3Z#kH!8$!4= z&|k;!l^G(qoS-!5$LZNMxtzFteJa+!V#y-jxKfxu#>N3~qi`*v4`b9%+`du)398Ew_x;H*V2p$h$VY7uL& zbl*FlYsVXblrJz^%Awe)GGHgQa(OcRTjBa}keHCSAHoHd%=oLST52n&H`XhExc}KOt-6)>r8TBwM=0WE8+frRz znR^d&o9=Y3IRtvNyI4EaP?kwRS#<`rfFwO934*NvZ>=GO@ zV_c2DQw_?2+6O`VUJ~7xo|7u!yG@5anoc_R_=D*_>rT#u9t?rf3)#$79SAoTCJd*d zdQBbmFjkY@Oy4=bj<2+pV6fp~no+@FCkMm_TnleC3jZT6!)vJ-?^EWi6Vq2x5C5?z z`r7X~{yL6zBCHm4Mr(+5;JKfa_QxxF0aCW9WSI8obJPihlHk8p{IFoCIK z-BeE+t#&t+9Eblf%l|dq=t!JBM-9G4_7*zrMj_bs zV+~3%@<{P~MqJ8)Jd#s$IG|Y)<cJOS-~Y{m<4yrw zgQPFpb36RVD&*4E2cc0%TxTn$!`Eskdea9n)p63Qhj*m$`=YnyomSAB^)vj=WBRaw zru$j`I3}jfM$r);lw21dq_w{<|ASBae&^O>#J(Kcpj(N3G4UCfEd7!=cgPfzhxe9` zu=_C*zis~$Sxt~NZ*YQ6e~U2_U`U;>IXcsce)Tu}ec#Vwz}_ij7KCJVFyw99)@I!4 z1HN(7+G9*4??b!6T0LQS?Z;-6|Z>l76cQGv3 zuaLjPmUBx(+ys`o+LGDF9*{9SegFTgf`!X)O>7i2zr{Br#c#fK)O%2i}Z(T zq|D|N8Cn-#41g~{t}RD>hBUjmv_K+~O;QD{KzTy`RJ=l0S6 zz9LRgfbhF=3msVb0>Lrnb9qm4JE{`6>7>5Jd$>oMWZ!~OEv?c14~)XUNwUwqIT;Av zBko*|pFc<-W)eZcH!G%nyNz0Dd6i?fFLWJIiBl7<=2EN}Xg8{v*(jeOebgEO;C_Gn zruNM%iOMxIBd}DuAG^(n5${b+Dx$coZbVW!^sP#|Bl3GzS>Q&(iLM=JUr695v^7n< z|EsW;E)v(M>J^5i=UkzQRN0dw$!FJV33(sT$eVAUYX`5+pB7QWb$K8^Di6 z8)jN%m&BtDU)u^8E4AGnQPlE)DX)0pCD#MH<2&LmneFDAo3DZ|N@JKaos+wJX!ZEh zg_=aGqETjiUKVUd!lfC-Q?|m7Sx7W}RXmB=TTQ}1C$mor?aC0sLrkzpp|mWHRvLe~ z{hf;wLb+~Cu<$~^XgbN_o(+p7=jWby9HB7)+I)`wP=Q~wN{FB&!86EI*T&+I>f47z z>CwytmawEmmE3HWe!RP2{&s|r#KJcZA)Sw1Vm$bM#%m9Dw;xt5bF8!tAFAcv>c{LB zE4cDb?c6O&UwxxNsn5WIqpUw%Vx|}rh>&=P8rJ7QOHAWhyb7&AUyFdgbiDN zV$FJMv|)Ip50*CSmCLDT*vj42%q$0u3iGS@M!**}>Zqd@!P4rETm8`~KxwIvv)Xm| z=X#DBq$i_T$KaPW2RNH#M=FS2xDBW5u%ZPai6|CG+~&<&xa480g9|PL0&^0Q3&6Na za2-D_pQ4BfnrHyo>UsZirL~hRg9dZo8+(4S-qFr%o??$co%R$YKAP7q0~i2MW`(;I z-|{tS))XnuwBj_1gQomcZ=+Pi>B<`~X0Z!DM5-M1bH$+=HDE;#yYnU2>8HFsX^wtZ zN}>n#XyS};uYpHH_fV->eRR`36p6-nuvEh$TxIlR(bX%Xw_L_6u@5vldmWg2>OyO& z?a?;#Z^{-dc*JxoSG_k42sOqPjUSRQvCt6lo&>FIKi;|C(ZA5~L{*c4Pp4o@@1e|w z@)*Ki^IhXQgmMXc%j-}X&xk&dNyhJv7&;<8!?gZ+<9|kf=6j}Xjp38XreKN_jDs4L zMaTzAW2kiJ_o?yTlQ$-L2AmWsDCw(e()e@C0AB=nVzOlxr(afJ<%t7cdyCc zFnv-#QU&Do3%d)XC5$u2L8&4-frsrId7CAIMW7kgYdIaiQwQwB6YW~`Hv}O5=I|ct zY$}C2w_ZTbkh&aeed@1%>jCX6*(Y`Hu@9&&IXlB9<{DNz=B#m=1HM}Z(ck>8+XC*` zdiz}pmxHOTv??lP+i=ftt<96v{ACf?$zlD@{qX(pzkRy$)Hb!7RbJA%V$Ko-Fa$BY zXf|${WRK0~fB{rylmF;=WBbdL;{*LhOb-tXx_&D}lDXy!S#2mVC zyi3!RYctx#ZcMM!3DU;Q*~JnrE{;@Q{53|-!77pRMVl<`{E?EBAj4xOwy+ZvK#TBWzbNfVj@C$D>gP!5ZpC%o^r}{KYr7$k+JxJ`%ly^*7Kn zeb=9y-o=}$Hx?jxcf#e?=Aq&>)-zrQbxRPikNS7|6lo=D6C899YoELy8o`e44z)x6 z6J4!>x7?e)Lw$>?x8$2#2mOcSoa}dtST+D{{1?9R81~r0qP;=eQ}IQoja->0fw2@X ze@y+S#woB$yUa`DvCg*pqU9p!TE1&kHIm-Tc?ZO$up)aM?h9T zH>bL0L9klCZa@6pQZT)MKJzD@x}6 zEC)pig`vI3^vL8$3>-XGSJRH9>)%7IkwmQUrZpaaxmR_p8CfOAyIH^X9s4Eo=Y;o) z_j+ah6MXy`N1vPQNcNxTRUqk3&iEd^T;dc zqG$iHg|2J6_)Rm~D!gKbo!LpoG{P#+s_ayHk#f;>(W=^~+D(_J?c;*y@@3&ZAi@D} zjnR$hk=QNSE3!OJ__FMB$h6lATq zR6lhqm~F<-ki>&|B3Ks;)eYN^C|f?`?~XJ_1tUmG3+w)(u-&pboKavbRF}k_yD@+d?vcB zd9b^>yt=$4x=u0GF#TBf?fnecM81NCf&ikVVpAi`K>`W1aF(#$*{4vraJleWaB~9| z3@*1idmVb?4XB@_?0?(4*sEUc8%SR5Zar@`5+_(po+WUTJ2@X{?y*}kux9~;&P}2d zOxr?}uV`3A0?RPQNk^sO)>L&kg32&YFl7^+#gXP@{%dG*tejZBJy|<-N!Yc~OMZy?3eI5X62pltKf~?o zu)_Fs#H@+o_t5s70>!fS7YZ=;6#AmEuC=EjM<=`o)2{R!Sy_}d9PyO>wH%^QNj&pp z*C0mayA;+Y*5$k8@2I3bskzg6<)4prJQ?qQ`UqN-bYR~Ox1tKfd`e)CC~ny~YL86s zk$7T>bfoCZ(|=hO!Z@$rTge7Uy+e;geFyLQezkxvKl`;{ZDQ#U?T&1_GwKTghjpl4 zZU*=4fk@0*pL$O*u~BZ@Wray&J9tM_Eis`)*YYe`AR#qam*}k{ z>nOjCr{;d=oOOjJ;e+iBL51U%?t+7{n{4 z0M0?{y^m(mQ?1sqtr=;<2qsO_2x5%PNqB>ptY$}p15ht$+x*3uIHUY>bN1lV^wms7 zqFHZpFW|+Yx>C*4uYud<%&!!B5rSBzb+ZnzO|%YlIl6K=L%df)w0(@mt4j{!+@9PF zKYJ;6@^5O9^kdjN`u2=Y@7CMAH?^yO9K0SU*y2&Q8x1T*2a( zg9cP6G>8hN9$Y-peWKTebtMGztw!WxAtXlv6RyyFLVE{bb`<;Y!2BEoU4Mt|l-&k} z1eS@E{nRYQBJC^a!|Kp)kv(F9N2?30d9r)5dQ;?M2=$k?hv|rSq962`hH-`9Po2oG zu4fGcS_cB9-NXMCXv_+#q?Fyy@w8=mWdKvx2h?lvz2f+VC(JaIInyQ*g{pt!E~&R9 ztPH@HtfcWf&$RUdR`RbNyRT_An=S{lOVwvCWigdcgevR-NB$Z}yL2+{u~g z4H-=a{|2_{N-fPVc;{16Iat4w`MqrufYhzGI-6L#nbmvArpRK^tHa&`CozgU z=qdQh$+%*zq>_8||8UM@6Xp1e&Hv>3(9H-GBE*I?{mFRALpnElS7HnGYm$I(k>4g3 zl%Ij}ktPZ+l=$+74!ocIh?#0U6R6gz@mRK*5a|`$e2f)OGuBxN>fsd zTY&FSDrKU-Cjc_zQIxGT-$NdL| zi_D*W8*6?nKURYPIF^W980MEE?FbhnXX;x$MSgVADAJYQS@P1BA@#=wgD7-L(-|C( zO3;{2n1*ysd22a23KPMLwd`82H zYx62O=s4AKSf7g&!6f}q8&h&ID2e=`&Bp2UEeW&XZwZ!!kG3Km7xhiYpE&TwoLAkk zndV%zpivGM$RsGWhrE?wo)c4oOw5MnJlzmi@U3gTaS+^v`Bx_eTd~Vhr*vcu#%$nF0!cP)KX2@ehkXoA_%%g(q-(w z-T0U_Wnx?ZhWA7?(gf?s&5F$;#~pzN)(Y`^&YuUp$s5-r4f<+f+KM$W$Nv5bU>FEo z7Q0AT5FD_l_ub{0b?vupFUydlbg1#JY*G&x#ca^7gTAbjY5%(`bp9v+&0)on^LTv^ zU}x?)?H!(yVguYY10KeQsvamC4&x~5AHKKr3Es=4x;@tV`2R4EvhS~sGzLclt=V&N z)|!{V%4e2Fi>(f=AwwEBo%=`F8v0LPOo5@fVIuI){#nCpRJ?(Nh>I7ZW5emk50n&L za+1AyY{g^xQ;&d9U4Bg2!v5Ug@D5qpFh4ssWnEeVnt}#s<{?D!FX6$W;KMOD zBF|e04a0Rr;6N=vm8k|J2Jd|Q;z|oHw&&R{C1O#q^>txu?1os)l)#u;am@|fAOd$~ zYY{CqIB~$kH8EG~S%+vQIou-Kkf&}@S_zhq0ajg6c6ZW@I~=>d8ey@P%GM zbpGK;c&GCoyN&AS$EsHXeu3G-yT$X@t+efTo#A&U?KpS*a$-q&F692@IhE!`J@B5t zxH?a4z#UuW0*2W~tZUW}fgncjAGBLjZXriZ+a>v&$Gqi!A9{aapNQ?nP(BfVCtI!t zBTwe0Sgr;!3m{cBW(Q!UHl?IBfeH`kw%!OK%9`Zg@V9IDAE0Jk6D-;X8#+fCe9|qv z{#mXXEUX(mwD(@s{de3LjydkY?4q-~s0QRptUAx8!EJZ_E=;zit#b z>yGfKc=h@2VcL9GM1f?X!|?-V6>;i=mVmlWrz9P;ClWYAvx!G?=%UoAGhoL5;U`IW zrju|243*UGtoiCNyJz8dN(!Ns*t6k6F)6gG#bdRdC3-=4{*DX)(U27Kxo z!5*%U`Q%42fg98KnT3%?a_9UWhE+9qZXzXc$7T~b1hUMJ3uy>=W^r`1UKsC7DjF9P%^@XW_NI#2 zuAC$^3{`aM)=5N{OLRRV#ggv4MT;p`%u@CYycmVeev-Tl%v_W``0PC<|zz&`BL>2;xn7IjeEN8Z)1JwAwEkg8rfjvw3~Q)0bcp>gE7_cNXvJx$-IDKV?0Uc+1jYE0VFLMI!YM4#9^kS5?gb4Jx>D9t;F+%-wpGyq#ONFs^tM#-Quw4i8pk0^3qG2Cv>o`>9INEKOk3rV2-r zx^`*?fnt#H*TLMVjuCpQC2ylkK`y`hfTv}Of+R^^R1g6`^%|sX0*|kxW4s$AN`aHkN|tEB zJ`UjrL$Y(siIcdGOxC*}rAtclluxN_Qz%P;gWBATq)6MWcfk+ejb^?fcK-mNy`XP9 zzHm-~dZm!qroHyzWyq$vsV64of);l)Ke*4{S}Qc-Y0@M1+7W%8=&!v6AQ z&A&9m%{j9S8)Xxzb6|YLQd7Or6ob0qGezed(i>YKCc2mrkfif$NVwH9p|B|=%wr63 zJ~-y<7GA)car-=*G z2EcjAAz{3ZG@9O76F^W2BcXBJ&}Y(Jqm8Srj8rDvD{mJoId6)Fh&4 zZ*=j=s1w8Rx#r)~^V;LI*`t{vh#HOdIYi+@U53Z{aVa)YVM+wgIL2>l=H#KYxN?J^3yQW7+mT68i#SCf+GeI z5x>n{^Wn2vp})uB*xk~uQ;s`vAOW=#=8>`)oa|#MaO*MYRuwRtnrV=P~INv);X;rf@ z)GM-v(Qu$%_U;y>i~frTwNM~|104R6`l~o5mpW(`Fc6u@OLwcYHR3{kB{dh63H$e~ z52a!{85Twv-+@%r^t`?|n25wFhiUt5RdE>5wgue8nE6 zIv!PVM$}F__XK^s7AZ57sQC-CKnbh2cL=LW+V$}}4y1OgUJWqUA%U%q>e}Fz$eWR+ zwA%RRsg9*6GG*pwl`cco$@k9qj$$s2;*hiivm@+`SCVVs-(CMDs~MBAmn-^mx5K*} zc!_a#^U2VTogI%jOL!f!(CaHhx0}|Yt)T|o{!|`7ST2++;)bq95S9{ zx!#LZJBcm-iqKTvDL z@>Sq^CxFLHCqF*PR;A+>0D_ z4QUf`8JcXR*Nt1dxIz)(Y&1(sd%VD*a)u4s!OoCWDY}$=dEk85@9@b|9Ya~a7)t z64IzoIQ>~u8>x_V2G#O-trw1HkbF{hz9v2dtgUK~b!;klXU`?v0ncCKy)7lw^)p3z zjO4q0;}s`LkTs*XH_9E&yF#Y~?M!|WPW+1ktYIFeJ$9;Pxu*3Kgl?{-;d)tGN)Ndv zM5o~5!p%SKJ7}*IJg_fpURKkC41}Z}(gfIV7l0lUgW6=Gp;jUlm*?{T(8E;qU)cmo z;MrZ>xmKdL3-Jr1yn-hO?EFes#wWVs*2t@+ z7QbbN=Q5IxEfMr1e(AM~{Yq54`#K?SJ{h?$fqP*ItK8G-b68}Hou@xNnnK#ZLs6c9 z>X^50kJMDidoGjl<6A(F@~YK8pXoHZTNFt_ry z@ZDpn*jpHe(Vx&O0ylJ?WzFJqGLJR4*SJT>M73ryb~6`5wYT;2y^AGH?pWI&++c#5 zacsJI#gRgZk`PZ(ZYiIF{+UJOi%U%%aGtla^VXi2 z@5npNsv;-cRIMA{i*peQ`Q7zNdg4P74)Vjr7pJRUii;JAkF6$E=$jAH+_%}XN7fGU z?rH*vKU{3t@-e>aH7x)Rv{s_q{t?u15N)1RXs$$n19DT?<&UP@p zIBvQ5c<0UCFiUpO__ssmLuUojAnCkR3I0|1t3`msW76whImzEvMZOx4nHYDcVB|H(!QBN}a{pCE$nUN7X#LFUK`G!K@9Fz#~DHp)2n zdL(+{CH59I@@UV`Ka+_fo)ce#Yeb}DI`K>9t@tJm2x=-(PM>9$^gXS*YIlHCdd4hM zMu@0if>LX6=$%xPt0UNGccW3*ZqObrt^l3t0qMZ*5jU~uKh7*$`9aYS{mithDk`o$|42C{%8gf4#f70J+ zF6DVDRx4_dD0)h4o&ir7_Ox~lwIQF-tP~H{=?>NTcDQwF>Co=e=QhcZ&=!{oA?UE? zCDjYsKT~}O;-aMO&Og`*g38+Oo@Zd68shs%ymYcUD(_&&mhbho>SVi{oqT~-=y*O6cJn|Qb=P-+;;pMnPITRo9yW>;2GbQWnhQth5Pm1Z_t9s93) zkAw{zrZsBBo#C5X@OQ0)jc>7z)zEkthcG7_W@25G-dpVRz$Y$q_0K;Ws><9wHJs(u z_cy%~t%c951iuBBQu*H*L%-#$tHSPROC=dy`KWdB>cn-QAGxV&Sh>XH|4yqlyVkM1 znwRW;DfN6g+EoN93HZPYWIS6-2>2j;oP^Ct&dshaH`m`BuU_drbe>UozX-g+0KqTERNdB&Bh?A!UgwN?pQgV?rXM#GeqKI~ zef*{z9cQ^30TFuLiu&Hh?i^In+3Ch9)|h1nnUPRcJf-=LbU5tKsM;yLKC)0*QCZc! z`?7YAdgJ>7GZW4g&0?H}@0?kATRB=ObMnWrh(3m(jW?BD`EOU|7po`3Dwxb39lf|4 z_3lqDy{~=gVfJB@=8ajzZPBG@cO|-{^Uwh6IEN=I>u{VXcc^R+ub%k zpS_GsVVx|ub)Nh_ZSZPa9uBE0*ukxw%bCueK3l-_5y?jO@w(^Bq;gcQX*W(e+cRu3 zYZ|uAT1Xj@71OVY-Zr`1SI!zq%a#L)FSnu+Y#`P+Er|Zz&3vl~`$b0kWv4f3Kf~t> z;@m~PXlkp%8?qA{c)c=tuEXA?CbUOtIj;rn+`9YMP@uh4yq*L5laLF~Kc0@}Pn`!t z%iAJM$V9X9OiCx;)HRYuF1*Q%gT)#Tuu&LD%NzAu3m>#+JKvL4(@Zqd%n4ZHrCA&% zAIrLqq`HkPYF(|yse2%oj^o8-M9bmOhclIXc?Xlt~s1#7X(%hfw$ za^iLKuc(U|h=&l_3w@S8bMN#0HSm^W#+ewP3s>Zb+gj?K;T>WeFK#dQ?J`vU0rE;; z$F>}=b?9wFZAgjI`c-8%+wiM}-x)vTp1EG|*OhYj%wt%f_OK zZ90s}uNi7?)NQzRU%F-o88Lh9b_1}g4sM=yvo&tD?zqK`W1K09lW_B8x*I5~+BKRO zSxGNrRmYpbH18Nc0+VenV%Z%af)x912;K5AF!`_E{$v-aV4hzyU>I#cUpf_$WYcXh z1di%cP-s_>^M)HIkJjatXbGH5&e`G(lHL%%lC4tJ{f#?kaTBkZ7SvMcvB;`Z5YKr9 zKlsyf`JUi2fH-A!CU^#gXG&(B7*)S&?Z58V+MbAUq^=n#RXTDj?~IGpDpHE*LQ_?! z&QIfU#@~zfq?SEWa+~IjnKN;Cg)R^SI>hlN0UbnmWPYThEM(k)-nl!J!)6`VBJSEx zn%x_3NEIV!?3?iv=E_too!t;=S3S6EwvxP&sBQ=X*VP?tPsuOE50!F0%a_m)yt7p< zolfkVS9&-1%dW%_hu_AUb!4Z#O}UC0Jd=j!T0fGLEw3YnJuPC?n6eT-$k*esy`4tl zHAh$X0M9Mn!i2Gx{~Tn}<7*PB=Vj{sqeupoX6H#={2Yb&s~^fa)x{$gZyF{5TD+EAC3}EjyLFEuD{2?j5pr$TPnxRao$Q?RL7g z*WF>PhS%87+a;zP?Jo zEYrs&6}_JRx1WN;zBev3EhvFK8dt1J-^+X8auGWa(|eTd!7{Ey@D%#$SFR$&p@rz!YqTZ(7t#8gmoP%;AAk}x5CPC*Cyd91#OPnK#q$Ia z38tBke&t%^W((oq=5K)5X@WZ{rbjNe{BfKJs z-Tn-Nep4}o>-a$Wud)pJr7Zi7h29PCeJRWOVBZkGl;!_m6fA5_?Ks%}Z%cOAFHQMB z^MC5fIzHCMi|XChuJ0)A50>Wx8fW(xo`U}7nZ~nhHswMrgfW65a`HN;izF(&`B~|P{5fGU$;ol)H_ zEyo#-s5L}z;RNCR*|F>-Q8Qy7%-8YZxL=~zVZKgvngP#NS3Fsryol5S<6r5KTIF$h zsjACObU0>~#B|w_nRGL%jV(8AQ}JiSoNRPD!+GLdhpJ`bod1P7!@^nzaNtnATv+Nl4sPX{M z_QzUFvD1r95#}7c!0@!c2VL3B=pCN$D{E=DAM?0h@l|qJcj;;R1J>0A-$vFvThw)B z-Z1=%&agHLVLuB%p}@q__xs$qxx}M^XPAGr;T~P?!EMLI#2q2HE~NZ#a@a_6D9h1Z zY$vdDsvh~=EQl04Lmv&H1ng)0BUE#IXN?c~jHiNWbVlklyg9rxrZb}R zcQ(W}u=MY`&_(Yn1C~>kiUC7HX2OR;hr(DwSi-xxuu1?B#*}O>&6LX&!jycaJ^KF+ zBXovk``!lD_TxXHKG@D^Un!zCi1cr|-`5AKrx7QO7BRoVQ;024l;Yd4kLZu^kNA$j z%)y*-*kCZ}X&BV6jnCdA{YKN7-5 zx%IhDxp27-Q|d~yeLoIT2im1OlJ;(8_Y0vHgeNMQY7fHvs{a%V$!DRr&_t31Wy$B6 z{;(DxGaLOyCa(s4MecVXjerBrRERLAP?U4gG%)=hK5N)xN_>i9itDQ; z?1wqjT*Kes=7lL!C^C@29!?}TxGj{?q(Rlf^deLXUTmp+M`PHvogw7K!l$0vfp?*D zf95CKgZX#G0_7QV=Agqi+o+S}Lt4Qw!HNC<1xrA*ztjBPXnuOe__@qa;>VaD!wBdM z(lf$^99f809B@C%%!1C_k5Y4{3Z&wWS>N=;~(` z?>P8uR$)rlTf-`rR1MsyE{o~6TzoF-^0F%E&-Y-3?^ij@YFizsJ=MU>99^fO|IBP0 zT}r*gb;*{l z-EBQ5deV$tU9YLXIanWqQ|{7#j-F=?EhoV#VD=^bULnxihdupK*%Zp`F5IO*<@K9Y zPuaAlyBJx!YzC52@|4YKdL3hFm(645c~9Aj9&=r{=q+=3X8ti>6w2-q%042L-7A!R zR4BVoDEpXD_Hm)?eql|6W1bSmJSB{IN*ME$Fy<*?%u~Xcr-U(2Vay~!5y-CD4X0IR z&iFpgdC4am75%T{la7eGqFZ--@v;m#iS(?&PUkkLG)@8f;oYw-SR)T}l8a!jBRFdM0qD)9FOo=T_g|;Bc z)kR2_-F2g%uS37I_dSY}-)NmgDvd!tj7D{_^HiF|+LqL1XmhhVm(gYpE40|ML*DW@ zR<7WapE9JX0e#w8PIlwKw^Jl-A2FZ1QPo1f;JDE$R5moBDGOGwv*3)z1UWxagBi)An|t-!#hi0^?8Xl48sH zcb1zIS;6V)EMLPqRP?HP+u1-1w%=*AcpdXGYn@~J=qYBLfCm?6cPNT7i<@XCo(dMjd`_k%q*TSzk2K8zB^H_oM8~ zd-Z7@+DFo;#4CurY3!2&$&`5J+%yIKO}w3!zVY&m^wpI`CSFZlgKU(AvCQ-MU7mX< zZ;5SBSr|(kEmqm*4)W@%jkY^^2_DCmF!u!PViU+q;?=R9!TBv?mn!U?2d#(OffT(w z()*y^W*NS;mw>ypEY`#qaF ze4c6m#V$a;iQ~6j#@Zb4KDa7qZyM~K<$W4@iTg5GdRzI?D9HlJr`H}`f}U%z*7l% zG`+zuMT{kAAEwclou8LS-l@!6WPL_5@8!iIn_HsYi=#9z@)EE}Mq2X%kNh|rRrB-e z;Oq;Q(FJ;Fa;(T}8=VgQ>F4d@^N#Rd8n)YM#h<)ZwS<+f3T>{VFC&A&ac*LiF8r?` zugPb)DA?+vYNOqt9j_p&Rue326gEe;zA^pDfyI-oxr6aE1zwW1tHkS!SEU7`yf+T8}qnv)*;{XZDRx1Hbxz&;Zzdf+t z%$8ue()hdw99#<3`zCTBX?LUf&2C#=?Bx>r%)!f%xd$E5EkC!3b(byG45YT+LeqY( zk$J*6LGB!^k;7-g&ov0(YADx8bQ8K#^2`=Fx?xd=-#c@(rZsYSUEosW%tRJyAzbpU zBwsPX-}2&^ms_?*8dZ{8mi@+6{f$nIwEKp;jZ-1Ln}O7A2W(Y@tx=J8*Xw&4JwP$| zD=O%Fk(MA2G{~Lhd2F*S3dY6F8(DaL{&yD{O%W!xIH%LrSD^760kIGgD28BxBzm*) z49DlOB@yeaBH!Z#Ms-P-Ue|x+R&K_R2*_gWDw_>ybFJ>TJ<|?0ICIiaHag-!H-usR zb{xs;OZp?1Uamia>E-$(nSOjg7TFaoA?c&b%0|7sT)zh%w6HGrO7e8mmQC<7SeCeU*;#9k+$Z}t_T)Sy+aA;MWul5d`5Tem0nOVL-GK00}z@btS{MW$`0lfp>v5(7N!=0xT4s>I(8roui5cQ z?ayHMT|^2RA}Xut-W!YxB)-dgBcq&owpqq9ToaMo71U^k^luSX*!m|F#WV@+${397 z*7cD0Yrw)`>$eP@*O_$+M)2u$6vg&ea6)MY59#CvDcvKHkWd8K zS(0*zZ!wJZ&m<&OI2238kW;Ypm*OR~M#T_*677fcQ>>qh0mS7Wt)CyvqDiQ08@=U) zIKwn!=Q`?1ah)c5+EWD)w@u@1Li7cpf6`ucmhfhPKZl5l3mYKCfaZBz=KWpRFE2kB z1C)`YiN9PGVm>woQ)f@w&tKEBRm9nLq30xoSjP9OEPuf)CPfWR{zSrKbFf~`7RlgnR8^b+hKW*NC>{LS%?khdv7vKdFD&6;Du;|C!J~A6_N9ChVd|`w zjT(GaIg(Jv&Ntad3g!x&mc+T4WD0mJ`R3((h++6SAnP;?7ZxC7_7L zAvNOA^>QwS&|p>3U6Sq4(-2ioOP|R#OdRa0vNs5%8Yy_5`zja<$+A|>awgYnHJ4bQ z8OwU0C{pZ=Tr2}kldru6S@hQ+%S&3-QPLf-3~G?xt1-IV6|E~}1}bbvq#6mfO{JE0 z@YhJOjxV>?G-yK!$GXs?a*Y|xVqe*~RWix{@2#PrwiO)e& z8hrXX>)2b}HAH%9jO8ShKy6UAPO?W4Wixq>3X6c7?1+i^XIQSn{Q8EnaaZYYUW3&+tWHxojQyMt97T1^sl38?OGY0wchYuY zxs7TIEfvJQun438%`M-1uo0Z3q8n;5K4>7uQ$&AqoF&6@TtgnnW|$<)Yi8G(qmt@S zhO%;KI~kv4?7X2yt50a7!nYJ88*M!6>sUW~XRQ6p6PtUqzhS7d6iGfF#lY+6)&9D| z>!;DDD(8VEU(Ss68f1s!i7pUO(BG;8nUMXc3O<*~$5xP;qB6pj)ZZ%Yw#m(g;ac zS$&p8)cznOsyU}zwvyS_uad%aQJ#I6W{9^m&j^zcZ8=0oEHNS~@G9?!J;u?GSy%6R zNijt_ch<|2H6S5bx~D4Z+qMogJ({`7d1R_pXI*JyZNZ$d+py)8q>T}xkEH>1}l+M?R!dozlkzG1-8xh#+j^rbGY|UG3~Hgt2kvv)xP1>&aImH z5JumHZaGlF-%zaq0aI2vsq76ve-?KGZ8&V9N*rZ!xt8BcS?th#2g?33%wKC2c5WBw zt9>6q@_4cw{c2}6q#K=dIvkcAxz;oz6BkqUn#$K=;i==&;bSm|y{TWc;Wig?N~n>e&wmg8Y09bxFpdM_qkVCEyC zt0YT(6|-*enK)c$WYBLhM7K~pmujx^2qf)7==*?ch28lv_Z+u5B13n0_e3q4=KCdt z2Eq~W2F6}SWMvn#wl`$j`o-QZ+{P)Uh+z16jw*_d)4I5aoWi)GOkEaU-hODk1jcAF z7M)jb)Su#VmsO4D+&&rJdMRapuh@WJRDjnn4y^BM6=&NGy zoj2FAoK=-G!d@KLzJYL~n)WWAG^1oI%aetyl1-3m(l1u%mQ@x#lz~GIi_B)BLmlXT z$=^%i6uYeJ4mi@DKh%TD0Tlcg<`k+840@)ICY3M`5-g6(LELEog3ZFklslHEi^mO z>5bo6ih%tS(1`!)I%A%%tJu z3~`t~F{dQY{sZYNdrR>8kvDIC_sCmEet6`^N3sL=^}KYlI`F-l-?{m{n|u0h?K{`^ z@`=AW`Js_$&)c^pdwyl$(Vp(f*G}Bi_tuGbPP~8Op}zgozti>PK>yHk-}ul@&wICh zrR$Nt{;{3DYkiOReWCBS2PQ`D7wOQ6TpIb!p1$@ZS2{h{~35;=!vmUoqc)e2Pa<}Iy(5)&@WC7 z4em`&4!$!~ANlmiQzM@{{Rbnzd-~|avtuugy+8CJOPb4QQgJ0_Tqtm&G z7tcO6_`vYz2IIq5Cw?^crz4LH_l!R?_}Ph9PQN-*9=>h-*@-tszBcid!G9llZ1}4^ zPYyqJ;pq#5w?8}l(%?Uy{?^0`!`XA+KKtE?|2losnXgTbocZ?TD}#SO_>+;hCte8j$va1X)c5Y$2S<~m&gd_U|74=;+=tJ6>*fc~yf*yS@H@lF@$U}~ zoN`CsA5NY5wb3t(K6n16vs>e5PBq5wJvDyr%cJk#`uOM(;lU`$xNaALt+IUF?7G_-n_OkDuV5S1!~K{0KhgWj=+onK zy|;9~e(ax*rF$FwXS%1lo!;li|6%mSqc0sDzomTqlgIz#*r)oRI9~0&zdP3b`TpNJ z{^Mg`>Q4^Lox0ThLjNawAMO70{!;g6dT;6)=-NE>Z=-kI((Hb<{|}G9H~LiXuXa~X zJ=godEq(vb-gm%7jkNzyvP)Zv0b!9Qp-EK&0wN*=X?7Mt#d5lyp5@NdSYGw4vFpj5 zi+X3*AXY5I4q^#5L{Zm{4Wnm2^; z+ua^xN8EDEb~^jxT7|8mj>1`iE9ThDP$-oT6@8U^l@7+9#(~Dg%Ckydc~|*F`9^7K zWMec~>1-5X9B-^K?qb~AxSkPV6k!x+)Yd4?=u4yiMnTGzDl_Fv#TL~@lL+N(2h@7G z!y^l8>tZ`#dCd8e-ByQtcI9>!YTnt(@}PsWIzSzTN2;AIlkr_9JI(wo&pCu!K5|a5 zEHb<8aMmo<@=HtHy2#`gvzHE(89bnX4#uT&xID@MySVxeetw-FlvNq=Hnqf;)<0#dS1J z(Ol75p;2tHu~&_EZ06D5rrdIswaO~OC0*@nI?J)O%Odq-H&0cT;}*wW_%`dajt8u} zxtzAHV-=)Yr~bv7a;ax^z;UPgk#$Fp2`-3Dh!tkzYBR?rO4Ul$z~-gNDMf2#sv=F% zRngumO;xP^N;TSMuqxfAnbkPeBAay8I-B{bOX>>>OXV;lBhx{~qmA#WryA2nD_yKj zT})>gH??W6e67e;-9`s0?x9i2k*FoG1hqi|K%f^Mf!s$sBICd~@DlMu#v-#3E3`gh zj3j_tNDD9?8HBDvTOnb{7sxkgUE~CM1RaTHqn%IQQnl7LhoYIv`Bjas5`$gdi#hkuPwCj_Z%Q@&pf z0%7@ZSi@4}JiHqNyMe#d3Rd~IHoOLdWvK`TxfJiQfZXjujw3tK&G6p&QoqEWO*cz+>0Itq^mB0s`f#c#e!lJ0LLaiao7h*(1w zq&^@5)PmL*jZ6R&Kn~1hfFF?VXm3Qc+)qy!AOTjH)`8QKWt~QyCE359{9zgf=7P!a zyqFTejLM_WPafzg*r)H22A~c49q5aeftKhpV&d4=bo0va?u;>HD z4R}R9ScEhM1acbI)!@CpOoLMBWRn3NBdd{k#2eXxq@x{BbJP?)geIb*Zj#_ANP=sM z#qhpf(!HY1tD#<8rG!9xK%mZcOY-amOSVGtf->BLwgdCfp2%`I&eFhspo1f#8;rfY zU~TPTNjFdr^+g>~Ths?cp-oUPcxEeTh?FBf@N7Gf2}`)vSgxL;W??Ro&id;{RjUSC?m1d;2r?UsePEFDvz1wf?H|F~0j0O`8)B;j_NIf8Z3+X#ZlObh5S`29oq^*z& zAc_4m0&+hsA=y>hi4!E~_N4EC6u${d{Qjmch%BS#vu@535y{TZ7Jn`vy!@Swh#w_r z2?)>(gn{hrm#__^I)FE+128j)f$;p6-oz4mR7jSl{8Sdmv0I@J+`59k_H3o5=g+Hv zx@5Kf=izI>!X4Rqr_O8XV++@5lA5ds=$H-q3C=lmXg5}~?KMm9AUA26@SDH`w=Mbs z1zTu0#Wu~dW!pge>)Z8bCgy?*2lMoud+(qvSLbVfOxy`#rti`h`0S>Ke!WNY)QqEF zwl2_&yjK9~H`u3Ne{>(Mt#v?ib;AKLM>wdr)fEDVQ-}2P2N%(;w;a|SO*sPSpNjP$ z^k{bFN9+fJ{;8C04DTtfcwbiHhgFL9Criu@$|w=aDixv|JeQ==W(Xq^7g7P|7E&2I z7YWU|#nhYWO9Ve=sjx!J2o`jvVAEik(0XK+Fg|oSWt6-hIu5}1 zdQ3g|1m)TGBxTa-G&ptahQM}xEEJ4Dg+s9laPGLJ5Vzh6bRJO)>(m#cVu#t#&3f0y zMt8Rb-IMGL^BUjc&^LzuyNhF{S_hRzFd<6uSUi9Q+1cWpatW>8V|lCuq3@0jD3TJ& zLQ`o!S}2DJkfuVJWIz;e(Ji_;3+vIxEW64_`t1rB?}#$)^-z|v@bGdnmy&efBr$EbQS%bsgyr@Hh|3 z`%6zc#mkG@7g-NXHTD*sx%kjKZutnKfFB)w*pJG&>JNsj4G^yFtxvB^Z@|}P0`(T( z5d70|LHY+RgZXpa8|w9bA$(qRsD6_wOncU+k>>d0M%rA{#+r?%8fzEaj?m2B+(bM2 zP^9KxvzuzWt&Y-k91_is%pmncy2kKb2Q|~TkBsG`lH)Y5ZQ^-6C_(czpTMKm&GpvX zoAcL&7Mi>tT56FCt@OKkwC2;Mx7L&`Z_S7QD@8wjU5a++g%r&M&s2k*uWhqy%(y>4 zzW?qd*>Zaq*>(+cjELeGF_L_&JWBGQ9q)n^=>Yg>2Y_iIc7xY0dYSw)ZCi1WT9;Nx z-#=d{1ZSQE%R?^;-VZL4gS}o+zwLcRPx{7G`1YVFu()k5B=)eNql+x5M?YH8$){>j zS>6~}x!qA1Sf?(1Gp0G^p5GiedbFg@{Lqr-F14a=e9@ZDo!dr84{6IVi;|dpQw_c# zNW-k^qhThDPG*jjBr{8l+A%}=w_`>&N@3hKq~HmcQkeFwQ<<*z?Qwo|dq%moJ<}+l z0~7VU12Z+HBff5YM?BRojls{QF;DPLOiEfOJnU&F2HoBnU-Y~)!+CXKVvlvj`<8cQ z*oEEjGezC-w82_tG^522M1P5U4*!zrukM9s#Pz~$?|;RFJN3rL@92Z`5Bo4a8U64! zd-~z}KHo4;e)t2*dY96&S1u+bTGqA8Hyj+GL*4Q z9me#ZF$~}8H-b5zHUf8l@f|)sa3oW7Y!v>Wd=x%t#Te$NQ)BSb!Q=3z+2e{U{S7m@ z#dt|)*~4Y5k581!liNy&FStKR!k31+FM-nm{GNwsrStKFeq3`!ne&o+G zzK%+l%H6+}konr;M+u8!CB$uhnJMMNtaTFRP?+3^SN6>{MZ*iN%==J zB*dHe&z1T7QbOh$GGEG@5HfyWvOwn3c%hVkzG9Kg=Vb|*C8L)}`F+_kCZsSjpW`LO zH{;7>KGP&*IzP{n^5eXgOZme6D`Y;)CBz3VTqWga7s;p|yhi4ek&u}fy;kOPn1r~$ zdcBlSij(o=y$v#-jyY03dIu}zd)=1tmTr?&ZoWrCJhj^vna`gjWL|}Alk#{^8MDlD zWj-5Ah-2sTWInMSQu+3rd?|nFvW(rQ?2`H1A|YO!vPa4b<7FJ;S0I(oOOue{pYN6V z4BRJ`Pdm0>%C9SzF>u8}na@)aGK&TulKEUCAs*M_u$1pRT*hBri)B9BO30+$KPu(N zI31G^>_0A*dp?p7|2E@<%;zBq8QZ?6qgt)K8IhoH`2^syh5-I-_ zJ1^mgybCg)w#R1%W|Y^Snd0*t%bs|G_c}hS1uhCV;vOWoYIE_*zZ?bAl-TDh)3*->G9y zn14(@|C`(XH@E$7Zu{TdCguM>%x$7S6orSNivP?6`4{Ik=5fvIYVl)Ow;SfQAn1E1 zi~iRcU=y9?b#`4y;=E8F$a_wGk0(r^(>D}T1D7172mE-9N*;Us?Y?@N0)83k9 ztzX&4MqAXkw%)&`t#)m5J56_gwe|-ed;KhnI@*Dz4w{4a9rzg!9W|}=PW-ngob_Wj zy5RA2xM4r#52Fi(-1p@lSrfk7%ar#yw!?m2g#En%(L+4(Z_;1Q^%tEyKB>Qm)L**p zr}g(S?5Dj8z}#E_FUAw%-V+AN0g~9BCnPUO-jKvw#b3PH^#Q!bRQ>ThzZhGobTf#mDx;u0NH>>#V0W?}zn+^$djd4T31jO#IDP zu^*xg#eMHOA}^2{h_Vp*gUl*m1169xP?0|X$}c?Fs9&&mOG#oz;pwY^_r)?8zAHri zO>&pp^YM^Te4ZwL4En3uzcG-9?;*{E{A9SuexnzCa~3tIw~e;6UTs7F6tewD{d2Ao z9~w|3L;o^iJ*!}SYbxubf4n&ho7SSUF5j=~+h=yKBfC|VpAyLaM&w^(EZ5uf`TO## z8qXW{E4RWn@*wSj?G;q&M|_-Mi^?+eEO?ZOZOp%{UGGtfog8tM|LWFNDmw56U%cjq zaIo#K+GexV)N@mN-oMOV@Vi?_+x?=0kW=iay}rvy7`ernw_oZ)na{4PeK5h5YDwYz z*dA`w@pkUKBF;lVLp-&Oyu5_%WIgS}w!u_bTtoir&=4?xv4%dMnoMu$*Ur#C9K0QW zAFi@Q#Z~IAt3}PSt;se|!2X_w^c?mbI{UZ#*Zj1IgW4r&g}nR4LO%7{VWHppBUr0j z#X9>nM}d0pQH(7<3PP716Zfk5U#3fY)mnAilfsokOKRP8Y2T7^9Eu4MS?+>Qq=(=T z;3=q_rTs?A+sIq69NG-fk+Hg4UUAs#*mzx$Q37_wBN4m#AQ3B2v@q!SN$rGARrjy| zcs`c>ZMvD<*H786#$dmLAvLNC{Q*Sr*-6!TPFt8>A9L>40Nfc9h`nh}&_;cNFy1Ga zzLn5WS8g3b7u5^Zl|Brm*BUp{aVHwlKimx0E!)_b9(XW9Hz}hDNY9DX^&Zp|444^( zP3#;EGy_TOtB4p7oYV|Ubc_YIfpM5`SsZv~m4KOFNT6+AB5mM9Rq-Id=8!~lug(hVlAj761n$Qs_*KUwPpWH*qg(Ms48^#!O z>L3&Sw?CM0FEBIBo%Lqi1wo}h&zqCa&s*s8IZN_uXU9NC7~oVehV&aB;D7tVH5UG{mV zE15V9XU~s!BYiu#6VJZ(AkD)(iISFH3!Edtg!wcOj`6yQ)_#w=*z~ogNoY z{`NSYSXn<&S*i0AulGia*sb%_F;)F#yH6QU(gUJDlVsbtwpP~tmz`z*tNH33ingE~ zBIiXE^QM*i11+mQM;cNw4o+Lz5)(jefHty8v=djro4dkz!Bw^yv5%3E`c&Es7dHI} z(X7WwvS;oaB4Kp}89xsYKGP7=dj!hH4N`EiJ(O&{WFyWi#+Wq?GU2-aV8Y(T%*c!D z&Ddf=#U0_zi96>lxb>VRaU{=*JiMkBv3@>AuA6R6%>2%V{IOqcVo)brl4@Z`q%>BO z?R@Rof$nw45Hklh8FgfvJ#u8ct~s%uC!HCSy)LZrrn=0XOjn}uFq}Cu-i>hT;Evz^ z+Jkr<>WLq2=|xF_Ka+kcfX!S|pBb>Z0XuPQ zAfuU05KNySCU{6NF(I)bW1A8}^zsg6o&|>y2_}v3+RovG&#iF$>8r+ssxSgScd-du zkB!9h@|&`zx+r|%!f5vHK$6)$DTXahYR1g(70Yf2jAKT{#S>Q_$K&%pw3+5H?`)=I!JS&y{v~QJ{Kp3X8+8$ zsFll*uXBlzhk5wW(jCNx@C!uQuNT;>4wvvwd6x*gSC{e9g{8!T%UAG*sXSXT_X*z3 z=qb^Qd&bnYdQJo+zrYU^zF<$P%9t0c%LwCpub442%84I~T$pns>awAXD|5UdPV^h^ z#vC$tXR$O7oVw=0K5FEJ7oPSazM~1IjY|-VcMQhAHf~6yhKJwnx5pV&<%6WV=z@*doX68)LtN&Dy$y z-1s7&IQR2T?)v#%tZKw=vQGXUc4lXeT)en|*w}6LzED|T=sSI-?)~ff8V~g|8PZ-zg-~B5 zmHHBYfm-}Ot^@xi`l>#@MEykm8GmMY3W11!(YMbdK2;}G*Fg874jw`ZbOTtlo4o#u zgCuTNNBjeH!F7c3#l>shN+kL>f{-u zS5Q9|?(*14fRyOv2!}fHfTV@g(;Z+#A^sEe@<;L8$Hq^uO@CATRz*L~e^NhH$Io=AkAsko zK;2w|_)nN`K6UOl=;ZWg%r|f2vq-JhIqs_H_3^7r?EaexPm13dnF<{EbC4w57k zANa7H8{dtUzjMCX{~6;Y1?nXO(p;#c)&B$?t$6)zj&Vx#sYxeH`Rqit-Kj)!*u&;* z$mSMg+KHA19c{_^ym&x7kaQyz$@(e&jCf!t)JYkn3aB5<6Yx?0>H7J^wI_Mv2>G!+ zgXNX}E&6lUpOrrD(r5TMPpFfgkotH6Y=q~3Ni5Jduf|wlS~dUncf|uQKEuDIL!A^s zIu7;2LlpP4bx-Yc$qW8QtWa*k4Lz=656?B{UT(A?-i)v$>(91ggF4sZv_mmAwwEomi(0&Sd|kE=1DI zy4+IQm7PH0%2W_kYP z+O+|!1y!FbO>e+HND3rv1`tHwupq8M$6$6-t%e-dD1A)tYe0Y{Olh+LoxGlgKxehUnimndCKZpP!1d z{JY;z6`wT+wsLA!K}y0w=wLb2a??y0eQc~@UD+@w%i`G_8p3Ruz7O6>bhVcY;Q26;gCkb z_Q!h3bE)$;{|5Wx;&}dLmo%R78FPGD%e#I|@HhU%RxSYF-@86BmuY~nObGnkeRQ_@ zg0}dq7D?}q|IbL-mG5p`{QbT4nNZfpq0D)R;`=4k?QJXg0grOD9eQumo{ZhD4YSA< zro7J84!V0p*pg8~QvhI-`UKJHA;DPh#D?^Xb|F}E?@;ShJT(cRcTT(x=a{;=$Zn z`T;{R{@O5Wy(ZO0ThYC?CW^4F>fg<(>bFu?PKRn9y9QlFz_t=$o2?*3{HSrz7mkaM&jrqWC|e?D=l9`6f%3mL$hByvw-Ev;hs4@fF>&(ll1f z`KwVY(t;nX(7y3rm1cWvm6kVIllE-=8tvX&*(Ek}*J_s+t}A&mY`u2MstqM)JLG8l zPGw7S!#47hs7)mc)SLO&=eQM?94IX0 zGmxT^RjZ2lAy*Hl?bjXQ+ZPn4t)z}>FMesF(dU_H2lc3mYvWsCc&AOX z{zLyN{=d4xb`A1PhjJ~1@;wVt^yAvexl1&cp&!3C?TG%q_N^bcXx#L@_m>R*+@Qyy zhikUYs`0{g*v1`5zrpq%)sx@f;m2kFEOe}u%eH-;OE!C$M@%T)LGBE{K$_mW!2RZM ziRhDeiS&4NnYg#Glw5uJ3K5gabC&bucx*8Dj191QPDUh4@fgRe%2?wyWu)!BSL}=# z<>aCw7xu=8x?C*d%3f}Wlf%cmv1iTQ#d)2Wof}R*ywaF;xzkwekK3onyRxr%;qQ+_ z;0<^WZ-CYJmYd%xV%7Q@!+N4&eK8O_iMnjQ za*56^e<|ZPh0#4NpweaX=3vix3*AJ{ z5@hCCVa%FZVB&lXn=suPeD$3T)~jD_kl4ugAVA76Vq-n|Z{GIOB4QAb_9 zM~<}FH7A|vNoV@k0vFxgjdkgxnXZ`2Fr3aA?}k;hcL&8^dtg^XJ;8>SUf4dndO+vr zt-JEZ8w^(Y>IzDILF!#UUDj@YnmQGro4lkx-EMOOU7xXmbjWPtQ|B1$!NCZSfAN#& z80^5~crd#D+qj!^R7!T=*{WHz~{nbpBMk``U7unh6{&ZHP+lwM*LMBez+g>=@GTk z$XHo@@K?S%25O28L=@j0`&c|%JysBPwnicA>^#(ylMlcGd;s4CUK=v<|G+2KP4ghe zUi08UxEFEev=`~(=FL9bCG8^+^XK?-PIvuS@*9709~VHJ>|LK+!89PYBm|OEX@c$I zBCVl_5ylP4FTh+V}~6Nd(YQK%#7eHi<+>`10%_$`PE!ZDQ6$gg_XOl+4?RVpaRz8go_HJuJm*+$ZKjt^3OJ_z2)WT?*8%R<= zOp2kGB{ic4_KKw^1;$ayaq)ETS_xD`&qOfsMIvRVYyrM1X+b^zr6ovwKc)ueBa(!3 z=^EO*N3xLDza4EBlOil^oeC}=O%>(mC#_c{Z}tgMitO)JZ}|N+VD)*h5X$8-luw0^ z9IHeI)Scb@-n?hjtS*Vv8n2FT|F^?&YCkbPk;0+M^(I-lPngTe7+3JVE~>^UwR{0@ z3(4LWV2-|(-wEOWBtzWiG1yDh_uqwM@7HCBWet4$!8!tAT|vHbJRts7_ZaA~zlN2F z|5QZ>td!QywAB=by0{<{3_Y?;cs?F_qkL-b!F+J@`c9!VXBRlMf49KR+yl;b=BP-v0HjaaD@;n- z2L|-oFZ2#PK!4xnpwPV5|6}h;z@oOce^>+rktnjGL_mg}VN(PwQK4)qLB&GV8hvh9 zlvG8ur5In`s0&eXw-|S@RzOtnsTg<#Egb)a*XCR!Z-gK*|Pe&Fz1Krq9pud^fq)S|aKjeD;v{{N!iI)3=--Fys+5Kvcm)dcT0oSIL*YsDw((AM=;;s-R0(s)>?wPe?B1 zDRFT(PhNjuK$Uk4o z+ho6lY%*JT&7@b5<~E3bx=I0h8m&n5`a+4+n5s-X!FrI@s0zV~RY}o0RpJ=j6WaGf zPa=U=CpU2#d}={2GK$gUH=gPZZA{kU7jB24g>ilO2}}Ay6T-E5x3GSYo4*ckI8+xh zw9+GnnCe4PH3Pyx(U8=yH6)r#jL4=OW8(h#{$x?w0OH&p6G&?jLhM;K5UL3=<Got;tOH@O zNenHT{Ojzw zZjB4KJDDf7k3pUFZVK~!Io}!ve0n$#9!MSVal{(L4yfxoNk^M>w>HwRHNFb()*hnQ z4@y+Yt1LcKx)o?|fh-htK-9BeJCSB%!J*oXTM?t{(C=!4tnYGc?|ZJgUG zVo%Pq#4ki!u>}{b-dC@1PU~E^^0+t?+-m{2XSB5}#u9;fJ0Xw8#Mf(j#G5&1YhNb_ zxWc%XHg;a8X&=MC0b{NNvIdO5&RRBaO8Dx(h*$l|d@Ahs!(@_1H{@D$>@aZYt?fS#^3;N|`=IOAM$D7Y*UFXN=&ZM~va!yZZmu=O&29 zJQ54~|B(Hj8aPpU2pU&VpKLO7>6yd_3WTdL=j>Q7 z=vW^BpTC}KMgrTV0m%S%{1IG*y@T}GOj$k2Fzz~RF*O&d3AqDTB6*0!^)7q^%12J< z6u?Kh0y?MV9!xQXbb4_SbtL&dy*aCxS|9g-j!xt#BD{nSi!G(b`j^ojpFN~ptbRs9 zeaqqCzK;-hQ3dQ=TY(s>R>C^Dl}JnZW4J!83gNC)!v!0j(9)Er@aZpjx;VauI={4* zW+LmTeN&#%dxM@+E4*IN%Vl_?XKFRV%hj5YZyK85nH9~*jC?8Vm;Dm)&1`|KlU^ZK z+aS2tY6V1Nu_9&pg%T~As!Z_;J>)iWTlglQ4E;CalYCC$7you1>R7L7pN~p_t_^J= z&cqgi9c*RsNcg=4;V=ONd_e*TF61)zgNdZ->D`p^N}Bv-#~!L>#$HHe`96yC*-svf z-!HG@2tEcS03Qq0YJ?W4HT~+kpX<8+)%`lD`=$0$eyiR+_4mwY9r9Yjcb)UO&bVm% z+MgG|l{b(BK)we!yKF1tRr7#Ew(U8aXnJ-9l;F9Y;8$Pgn(W#%>dU>G$hn(4e{y{BcdwD z7{7JCKk-xA06cAv2@fqo@J-7G@}(iBT*@3X{*tRXw;^Bp%s9O;S zimkXImDa?noP9-9rA-vv;0(@>ySux?;O@@D-QD5g?lQQ;;O_43Ft`uy?(l$+WRXoy z7Fm3Cs_*Sy_G$TUSJC77b`7P%DmD$mly0B})+4ojQaY73C()Poca@x>T;(R>eKl>y z@-6WHo^9Wop}_P;QB>jQxqDy^iv5bnYFUWp#*EyPO+%*oFG%8_YH2(b{|pj6ZTWq;fi0ubx7)6$A!9YSd19w2U+Wdsg2{o z*&rZ4LnmYEPg(}Hh6`MA)K{pUOK140#+HczE_#7-X%zG5(YxisjPb?62aM&MXB@}+ z6T(Nss7s8gKJ!)gIZN+9=%xc}h$kVjVkPHnDwvF#+WrIRb7H62qhQU^7NA+w$ z)moehUS&p6hs-V1Tnk8msCSkL*->|s4|(YVhU+BlocFNo(#corpPqsgKCsJ8;oJp; z5%^t=@pS8fhI~|r2Ms;!!#z*z=hgyfLU&i~m8jSY2r+hGHRqpK_%Z@*Fk3EdQGLg! zqY=1|Xq(ABKg(6iyLfX3topd%z31_;<+%^l91biSFuRQSVw9YK|Da^wKDSR3-$h7u;=E zJb@Gzfgi0S;Ze^I@*J?Y{a`_t1jGs_$u80rBz#XA|MVmAk1+Lvgoabp zxvPjQ4r&Km+D@IB7#^9)RDoB4nP5*U{U~Y6*v7~*F*keoCKsUBPwr&S9~-TsA(gS# zv7?)pD5~zW_-t2yDtX?&f1=xG#X<26etYOyqSp7EX}l~O(=Av0#WhGW3kK5H4^vUD z8>OB;z0_&J$N`hn;j<1S)Tcf6_W`Rjx(|?Fc9I zy7p#NLJ7Ik_Qq#~1Q2>rzEJu6O0jx`{u}Pm62v#aVF$)EnNupep_}hsm2}K=lCy`K-_5@Yh9H= z+`Ov1@}yh+Te}7)_#Hr(ZX{x3!iHvY_u0&6>}s5vWkQ+!;5zQbb!mr{*W)df!?sS!CBSSpRXSx?myLpC^Hv=0)N1|Tb@sUq$>I)6j=1glkaP-=u zvdfpFx6@D4=jd273J#RyPXnudshSStX=Qdba!TCUJa{iaYl!*?+9M3N3JC6L33;4{gy$0lNm0lNtD5Ik;Ix?d`q$v{@d^{c!et1Z+H$lmj}(c>5`q zSo3sSg64)*qC?5C#;0}Kai|x*ROOMiE<^S8j4DuxHex{whE^F92^DH&T&IW5trfA; zqQ&7gZJG@94-xIZ_p~{7AL`PBZ;8~~FCU_`2_hvz=!po~s4T^X5>ClXBzQa}u|40V zOwN5PbT562+lE?Y6c-G5Nw<0{>S>A1Cu;g&x4M*B22CVPJ{02myD6cJm^e+o&$m}J zmAkNyfJ4Fw+KS$@52K8mL9)r+!2u`HUd zCVm^Q-3^Y*X>}UmIVo#eEcaV3>2WP-v-gt$mlS>55CmYo&LM`1l55_2%7-4Ew`3$0 z7pYklFUA0fi(LkYE9v5hoZohl@dVegP?;}X4&5`LcaK@Gksu#=uxC(sQB}OK|IlZ0 zmD&n4!`FaB&8QN52H{4YgN2s}=k`T%j*yEF63)*N{U7AGn-1PV+>VDYVrgIpEl7_P4#yz2!hCktW@X<-qL$IqiQJ+-7#E zym%}Z#cgUu+i`BGLyPK(5UZERC}`)0KYW}cW#+0-NH${n=d9d}wKqLa#?IF@s=dZn zrz*<@HwPg{-VJm`@7={I>l@EI%Adq+yfZsfbiSg8LS>AlY?`mjEV}4>VlJSz-_y#0$f-8h;DK814Q!O`*cBXBgJd9tfYN7=h z`XXYsHenhNXomv@4{-#kPIx%i&!Ge-3UU!!LGj~>U(%rGIRiVtm$-`+9ijF`ud8#d zEbG}y4~X7Pc=EZQL0CNFzG~bcAHQ)c$=wB zwv{@ZBd+v(Juc>0S)M7Ec-<=RypVPq*R0FHl_G=Q3XAS=4%Yn%T?X0vep`eFdd(H$++lS&h*jtsRCMH}nFY%|OaFUF#xCI(+X8H>SErBNlZ zn=4XnT&N4~)I^ZJ7ID$|l;9HHhL0Tl)+@)92K2KLtND|e1GhEXhS1gMr%NHyeM%@J z-U7x2AAm%acLB?rvW&m&W*USN`8K4>2f8o@8B9Yznf!dX z$&{7_o6AGg2k{et0KP;`F@yj-eX6>3XzvGQ{7&p(A2~H}Uk3WbYXis_gFf>x;mo zV)Nw$2GICr0i#l5c-G zGUDcl=ph_I5k7Bn^$)z z_{Wsonf2u-?r&tnr?;^_(ER*B-djs_L)Xjn)_fz*6i{)Wo=G#Xs4!2Ti|*5P7%M9QF(nRd+6X zbOPz39A5vJa{(JF+vRQS5KnA)6SRMM!0VcR(CD4U;c=b^yFG3T?(WnJwULb+x~#@& zT}$}OU{PZwY+A=@_^i4$_V&>Ep#Sh00>jh&;f%4m&)AmvCH!)nahe{1b$eK}l3b3Em@%f~VcHbizVt!4QkO-jstD z#U&J4W>U@DjsA{K75awrZ%iK1cu5?=WM{nYV%EA3}|!H zBZN!nqmE(8N_nIGW}T?41^@DD<`=gAGsv{)w#?JKhLP&!7jo8{qn?ytVV6yPg{rcd={UM|deg748Yucz%tb>F7Y+#u7pGV^^8)K)_`&!zVc4b*WmD(Lv9=rIjm4z6WN>_1< zbGDerN;Wke7nr+bDH!qFXpI=L7&^bTi!pl3zj3V2u9QVwyqb&FE#mf}rZd+#MsxgP=P<0Do$^m0VGX;CH5X2Xhgw0a-6K@bAl}ws2 zvZvtVnW{GB32#pcaY+YA6Mqx#ZzB97ydA^!h`BXL0{=)O|KOWMe-o9l?-6iFE_6Io z-{hQo%1)d>cMQ+i{ybr@Yc?HVXHWQ`qZR)NC~f7fwQJS!P$)X*2{^l>y~Fj0X$E^m zHLd2OjPN*q(#70cXRHP|IyQ0+_m zLe1ym1UF1Y22qofUGFXCwB8eS?0p}cvY%p}-Ea|G&+tp+3c~KRq$bwOS7F$q zMAM*D08P+@(RUfNtt$Yx*IhnF_Amu=7X3^~O3SOneK^*$hE)QS-A{b6YXiE=1Z!q}rArf!s(n zt@S%1V~=4vjWemrr(g!nO7Jv$hF%`5W)Rw)9rCKcOTmqa73J zlE*nExbJB%UD!f%aJ9kH3rNxz9N*HbB{9M$+wSs0~zXrClOwC*xxz3%ZR{G;gGimAL+?NRR< zN|1U4rckutBL&FMjc9I7ZEi~!d;XKq?ZNHi_rNq+q%B(BpSngxO8I39x(WSU;%ooJ z!m-Z&1RPOC+VnrF2l{Dh!!5B^2gog@7!-T1cU?<3;)zNMg!Bn?BkTf`*`0+#`cI{I z=TV7Mv6+sIq(v;RY8eY#GB0bO5@hw4wTEMTKhCEKG)~Iy0NxxGY8k$l-%)FYpvjvi ze^va;jT(+!rzQ*^3@9X%<;B0bJ2P26eqN;-`Zj8z1xq#X!hZJzFVR4Q2tFW7pyJNT z>;~6#sh#WL;2G$-wfgP2*Vh*z)FumVg&SgI_lD1Da>bU+p)0DmI@-N6bbC$IQ*VAfZG{jjQPe~(%67x-S<9{YB{ zzEz>#v5|0O0gLR?o+UboFt95ewvl>0``;rnYL z9m{rMQ~HUf{*mcD2-)}v8i6P&cz!3AJ@rX}s{~-n{66~#FjoNlosJYPqQ7CX*Fqs$ z)0R$SY9Phjav(<}vXR$qr%Zr;YAwjoT$gHfYorJ({)Eq4>EX!fO*#Kyq3A<>ZZnvF z558K=FMJMP)c+glNyr{>dUjsE%J^VhXs6zcJGElis5g-p}HD922-Mvjeg&{ImA7k$0A@@el{YA3| zzr)RS;rvG)*_E+-jcVlT;2RX4wSENx4b9R4{rvm4G?D#h;KHA~jK__CmFcR(xwSPe zHhv$5+sGzV4}*{fkNeIeAH;c;UaG@Lot)8xL>r_dsnX)d{@b)YY*u*hH(8f^5P6p< z!w;+F{CuuymS)M=hk|KZ?iOBEcaj?d7k6C(GPkupYd2``zeO9K8w0bDsrB}e8fFX@ z_WqrS76#CS4>Q8=iRFCk4a*P2Mc_6I(K22o6RTuS46W#pM=@^8s(spLlSe5ks_0J( z&CnuI(xELV)P<9&{>JlQzUVlmN`X(yH9kgAP>;(=9|?06V9;k~&bhP$O7M?p$GsIr zNLMzJK4&*#;PbyWHm+qIP?){lOxnWvg&rR$`>h<{z9ag`dx2Tni4HGzPAj$75!oPr)Qy;Ni_Vy?O4p~(Q8@1{O`TT%K0%u|C2iBU zi0%haBJ2CM$fzf&@xn6k*&)1=FWO9@r&jOhR__r^ zQ13flUbxin+S&YvBgK#RE_K1~_NZ+rjvSvFaQ`^S(-k6FSVD`bHs~yBUFdXH`OXZQ&yQdnRnoeA!mC!p z8ah1cn)+1J+vcz+@{y>n)YXWIH9L9A5AyuD>LwDd>92?=q*e;7P|lK`CkK%@7YCV0 zUUn1A7JCIR9p9-K;1@F=77UXCw^-hzEWs;fDZ$|r5RS_EUP0bl;I1n<^Idpp*G)eF z5i$XrFMS9yS?UT$A$W@8mz@f;?cN^&Jak}{S<;pN*p0h*WYDYBO1`Y%(GHn0SSoYquBI4VmBDJTk zG{c%H#9rS?J>#1>Jst9Evi|KDBFZ<;QnIJCm7*t}36VKT$T9u7X=QBjbfnKwuFLbg zz@7U46f{thFT-Q&@35+3?nlRhevHqY^hMOs38UU)A)up4!gTX}j;^d0KK?zM)opqv zYJT2Q{;FrMqZ$Rn-mv;Xc@xNTKx7KIe$$0so5`v!eHhHoZ~(IL3Ez6LuS)Psfv-w0 z3r#6}R$!~)@$K$U8q|L-x12Oz5lwa$3Jak zDCsl^dIE&Lu2$vgyDkab9KW*t!cxvpuFlns3(m!ooZ|@K$9~>;24%x9f~Ye+1?n@2 zieD2aUR%tO7mIG%BZ;MnX>J6l1lkG9P(>LXj%Qh<3T;@}*2@p{?u@#sap7IBDN88L ziI=)78S9j;0e7{Tjv4As-0GSx1pz$YH5-s^SmAo#s&5Uv40ZdFFEjg06CF901sMiq zug)=v0&q*fLjFY)CG$f8bvykm!JAk88v#Lmd`wB`zqknW_ zm~<~~H8^SXbfaV`IhgiymsFYVy!bUF=%%D&=xrtC%)EoMsnhA0v29FarO@tbMu`Mo zyYBEZPW`ztd}~R(zGH#9z0DZS_RVLy=xane!DR2EHjbwLmWXi|$kqLc9|auT4MI)O zl275w|GGX%1TjthBjTaf@}6GJV>gw=tw?Nby{K!EE0*qV7^QZa;d(c@Of2@|FyRJo@~4Ah(8tE}#S^y0{VW8+;Zg?Rm#EvBUkhARDj_u*dupfdRVTH~c{| zOf|s=6(;k7Y5F#a-33B;WtaBpn1Zq{u}MF7vf!QJYj#_(Ry3dblUm~~7FzRc?`b_P zJPl|9eCi%XAY;4RxpNTyE+?Er4Y>O2mmgd0z4;j2eU?XBu<;eY^Ra*Y2%mL;_>1%h z%{PxN#-MNZ?F}_?ajP!&)C1Gj$lMUHa2!`K-s6HyQwJ7~<0OTDxXzoqx$7s!GJHbq zvcwp;o}QP#XEriRp0semhVd*kYZ}(6OSv-DF6XWL63nQlI+H!?D}FpPgl*jdcBzQV zx)qK$pW|ra>?nk`4#!3j56GNXbjV0-IXrp@MTft8&O(eB(2~8gb`}+>ABd9MT#Vob zm2m?V)Uk*h^JIaN>1oXF-#XW zpGBK^?Oh4QC{-{)TLiHXSq#~{#!IEiRG9g-YXQaEd{v2c)EV2Uc17xFyg%`QwH9St zVM2!yIj=YhIYk8#lCIk27+GMaq)dMDujv-ckye8}-(en6CbkB0ogywdlI>#jJeMSQSA?imFfw=7idWad-9U!_@TZNG++!QrG$?zP=W1{hRS^4pFZd_O@FLjH zX}#hdRyJtHdHb1Xcb}uuSiCp;)9?GJQo|_n26@-f zb-t446JE#wM#UC`_2k%PjNu!ThKMjKzEjf{xyD5Cvg(3x8jre~=I*kR0xQGFmj{JY z&^UW8#W;l!@iQ712345}vGPRTk=JG<+*|v!?%uHW7}odn+A9n*uGnXxv2Jba4a(hwO%HBgE4J}}*! zivpbNADtuKC>*U7>GL^&6CJv-f4T9&n?djXKZ+ot;IH$)tHMReUCVAi+a5EjqH?F! zZ7%+6%QN?@OMTv7Ms!X{VJI(M$db2cYI`7Q^?z;`AaTH)0CBPYURC9fDkzOF zDZM~4!JK*%Tk!8ut1gXYmJbiS=)HoZZ}xV1Zy&7~dw<;3)iWL1z3U7R@POcfZOMTm zTSr6l+U57XppAd&?(z3%x5K73Im5_*Z|e_nn>uAHZKI!@0D>kmBQ!M3OS zV47mS9Py(pt_b*g(}jo@o)F%;27fu@+3u43PIlVAYKHNgAr#8+Rm( zilqeP69S;{yD#(t_j<{xLpMZw7m@S4iKS1zPsGA z!<&ME`dXoN1veahHC)%a39?>tncC^%WP8%bTN69k{b?%q{1=D8|PChm%z*<7gUw+i zqS$n=4?_|>3+{l)VE_LPl<4dZ81IE9P6_Sl9R6E>@xRdfsjE7=OvEYvD^@22CABWB zE{q}Mqv6_7yiO}kf#XW2oQYQbyc1IlIGh^((tevv9UUbb3}7fMv94S$ZpBE6nq2+c zq2ZPJ7a0Y}rd#-{zd~rEcE<;?_Z4v?=OEYRC_BUD>x<2;b~uahOPsT*W_*qQii7XI zgICl~H#tuHX!ffiqOy~6tm~%PO!v`3BO^~kdlbW=Zr+yzKuy#-rlFc*q?yIBq~AC& zTL4DMZEc>#%OTblfgTz!hKM9#9J&|PRxrzeH5?IKt~$WI>Ns!*~2ldxv+OYWpKr?qyn#cV>O$Bz}~sW zV*#$<%%R@*`&tt{`!V@p$Uh?c0F$&5QHzBnD z4&x(@A?~92<00GC z8`O?!}f6RZL()qiS;xIUHtNqXCTJT*EHmnEh+oXZV{Dl`1T99zfrf(kauUd zzsEA2V7!#(B!}F!`^;&yVK<=8e*Zr(897M)^f4p+F}4{d$I8Ljcn3pzZ4dTKhFKY) zhwOgPJCVgW)fE(fKUM5=>=VnRL(Wu%{yic#P1~`yf7Qg-9icKFGf(2Y5hc!QK0`RK{= zu`MiFi3c-BRcNUx%&)p;%gf<_g5z8}?MWuP@L-Lru}tlI-E5}23*pIE(pf_Nu=}Nq z{b3FBY}GWl*~V8tti@u|@A&S+`io~g zKr?r0?Za#{)wmOx%H`ys=TYYUgMt=KW8AmVDR{zbO}gM3uw!W>ez)CZ{oybVycN33 zJs2a+Ju~<==u_1dl-rgijCnBie=qd@KtHdCZk231E&h>f1y{O4*T(R7mpQ#m-GV?5 zXqtn`HVjFF%ym>k=%un_d-u07*3q-pg706bGI0N7pGhOUqm_+=_e`lZ?^22LUU;9I z39-Bq`pD5@1Eq?ooJ@`z|IZBWu44l4K**r*87vwS%>NF?G4DE}TwwjbU=!>+`~M4p zspq2H_>dmI{%_C*^PTuV@V~nJ&u0FiBgz$6KLN^>u)}ICsd{ks>UCc6=SfaO?)`t= zIe_A3Cq>(Z?KWV{2BYnMGtH(8ShatW>4gvZZ#zHwTLZ)pOj$Q_kgZBzqBbK&vt{B<0@mR5-No(!XB z9A_b1x6jv*1?j&0Amrj$u_*3v4XpYOVImY};LF{9hm-$4tB<#Gwi zln#b#cw6_$>saL&>pkuUlaK4518M#vZT_d%xOy;k-y9E)mY=q&Axy5Xm{*rZ`Q=+eg+RmGs>I3k!x^m^6mQV!>`gD z|9l|LeP`T!P%SWKbglr*J2)m0Xs=gPSAu<|UkI5j&lqqR4(K1qLGB+jv5T>uAwy|8 zWl`k@|3Qa8Du}dT2Hx-xVCD~$2-r`r8V|6#znIj@zMj-2SdI5;Dtu8ZUZ#td0BcKGW=1oq<^kmWS*)k zKYi&j1G{liT`s9C>N+&0@Odu5k1pNKCR#fK=f$t4k`4)p)pp0_GDwWJvG<|wBs|&r zNdyfW7Srlm?jU}O79&_DgSS%pA9c#I9vzkZII@%{-gl;k)ulr=my;3cOB^nk+6pzw z_XQuWGa2kpI)OE}uq0PKLmfufmJAjP##UP5!n6fZU{*`u;=56h<{cLQhWgl2NCgHKR#Zdc+1h?v3+F+t&S$n2=%7bn6`n6Q0spqsPtsNk1 zsP+Z*FzZVzMQL4=XQ5w7os8ujUUVsiX#V5%Sv3^x&mtX57GvA=HnZJ=+f07PU~& zEU5f}{30n&O|OZFJ)3v+m*$|uNL=ok*(tfk~7KE%HT&m%PpqvH0VTp7t5`B&QLY{teG6j zb(SfMRUlcLhs|(8sp8*^p>buJgNJFsjJhkf4fh%w1Fqpl)!>-5fGn4DpB^O zz6e543TlVPR(cL_9&$bJWY$lc`v(g?d!N2Jr9}d4O=Z{zcd|0jv zxS-JDer5}CSha(YfyE^w$01KRj&6cDa|-eN+x|rAEe1HRFiEmw9v}Xtf{#ja!q_Q>c)UANZ3B?k zpb4lrOMQFYC^&jvS?PIo(f{Th^T^4KNDOD7YCBx3cAR#V2rE`Hp>|(TfiB86E|fK0 z7L;{VQH1sS`;-RWQj;t|v_)Iw!n9%lJw^p*2wfSrW@djasvw1lAxzS!l}gh^*| z47tYPE0}&GZIU3f*4_HIDPejw=CWe{AZaMIh-Sh)7NbD@(hWQLVWIF^ZAfJSrOZe>e)W?+jB0eebxZNz? zjg1EJrorht_A4&xB}ENyc^+p*h=zzgtNdz*9V_bWwW7V0esvQUjL#>Fg0Frx3MjCX z84EBRgqFi#xW=F+T?6%f{LhaaNKq#DII$2sifl1d~gZ6eA@equ>?7etn6-cR}6}`AsdRHwF zL7HjOP{rm4s}KsTz*%s>?D%SI911qzPcr#Df$vaov<7+Dav2m5hz%jM5Pm@FGV_N>xHRqp|`1`{%d56RV?pf1+X|egzCeCn83m7lZWZIayPx-3y z)Euu)w*E8gk(YDZDEPAdrwwuRna7%&3+5C9B(vUjft(jSYr@9F3d1*93luziesz=4 zR(#g%^}H)K(3bYimKhjV?A-QI=F1E`)QLI-g!&jqQEN5i=EgjuK>(&sAh@Q=$GinQ z_kKgUfdT>qTekkAiK?Cp@wH($4n&N~VH9haZ;{G24W|#fOsvm>@TPVY6T;yW?g@l7 z<%&!{JDUOL6iP!JMy^-j8 zr?({@%FV;1pRwCh2}ns;^*s~Oucb^Y6et2ii746VT8e+XxYM%n<(Xr43C4HZ!= z{khQ+e$ObG}*R_`s@TbnF9I0g>rzr;se2JmFN8Jr2|9PqifrTialIu_|-dEwV%|E_Kr=R zevR=5h%N`fNNx!;R z3+1@UBb&Kp6{>O7Z2lN!`!7;mWZl1o!I`eP$VR_4+(E(F@Om77H`kDAeOn~NUW=!Z zOebd^;ora$-#+-P<1Ji8ZVirf7paZJQ{MRHH#%H>BWIESduC#lQ&W%A&ThBj0~(Tk z|7Y2LhLSFmPtnyZ*x2@%2GK9f>sVLTy1XFB^1cJosE{OjYmL2ZyS4p z{{_rD{K`!XBz@Y!{$THUMnCR6hA%PlN;PiRFmpT~q2BVhHNfR7@3=oc)g`Jj$K_IZ zLa$jldT8DUhK=%sjq)Y-4|AHid_doLo#s~UlBzZh`C6Lf7T|2|*ruuj7x1^b$7bR9 z%#;(<$f#l0%(ee1+o~HTJLvTR3{~^l^t2rPf7|sxN4i5b3qL{Sb@HNLU~psFvbyi( zEMeWoS&I#jAUn*cU(5Wnmf88T!YCM|Ms+K9jV%VywgTIi zmP&%3#7p~4{fy#J2Ull6DkfIkBdqS>BB?AMC?2@7G4T#@= z-@WI|6W^*F9Bs7%Oa04%XX^(6I_VKKEpjnuf*yb1koXE(Wpd_WHgxAUZ8aZ^qwqQ` za`-y@gJ9e!*U!}4V-dg- zO&5AW8o^K(i2^VTCT{7|5w#73jcM+$4z`Ufs2RMm7rlUOsulPwYlLXNY(Z;34Pa|# z84M-Y>`=MKYD>%`Z&$tD7m&I}QIy%c6i2E{R!YU(j`&lH+<6}OTi*g(X6QU*-UW96 zR$84q&)@1ZLmv@)%IO{_adkbf*Ggag{uJrR2{(PVcF+*9ayBt`2liborwFEc0xfv~ zK4i=t4~%t5!*g^CU}nOwEP_`V-gm#u?&ikcj@>gwv|Xg)6$@+K?(C!7Uip@pi(MTh zptCoH5fJ|vzzLDt5_Aq}KYdlRA)@2oowCZ!nR9`eCp_h`At89TsxXvwMH~&8aj}WS zmpDI3_#a;3xzuix_&+|DDB+cde*r#(5D$e{%jHlxskMrjl zJgF4`EfU{r{7TAB;)0W8XbBRL)u@MZV@WZ(3#)dz2K!GiEP2P!=?uXb3#Ru)9S1yZ@P7N8aUt1f{!p$t5ds529G&&RJ~axTX)u=Wusq2N`9AR~e0R z^D4rt46UIL!&x|a`-24+MP~A-Edwl(tnA+n#~((;Cc|PvEdyn^B=*iO#OCSmEP|Fh z#NxWV@#4lVf5Y|~kG1IhvgrpBx$!~?%%gW)@xtVr>IW4w*Xfo$JPenukVYB#ac9!i zBkL0;<6hEL6`~xFc-YGMbn6V3+xFJrx6qp#H=UlvkcXSF38RPH+hlGoMZ4-tuJ|z{ z+ky1MqwG=`s*hgu|1EUw$3IWO_0f`GM&_s(CuT&y2ne8Uhx>n^32N2)zYDer__FG? zuSg)dj?v=_=L{I>RzU>)LkcLdaomCMvN&7P-7n|$%4n&)TShlcEuzZ#DE}`_^RaWV zyu9|w;QkRnfJlIGFkz)N7Kamj8#&N| z8y9KzguwRgkn?Etod9De#lv_+Xkk==;4+5K+*Nr~YZHS@+b|zCH*QaUzr$!M{YtJj zKfVra(EH!$G-7>Be9Vn&b*|P~QLd*chK)EtKAjz5bo(1GUld?3n9yaFxP{}NkI<|dx}rs-F6 z#+6_|A`+KRXV1)J_eG#UL@1DWLMVntLMjqd96-Ov*e$CgNvaUS4t4)F(R%sy6jbTg{{>B3kj@K3G6~da1N{z7Sb*HS31qV#;Kk$w$8`!Km=)x4mhvl2x5egs;;baPd?FA@haR zPpzsxd@ONo!z6VwMPzWp#(2nzt6AKvZTx<=Q$ z;}k%1YRV3i?gl1KZqtn_bOp*{BQ4^QJ{s84` zC^J-8!RlC`(M4n!GyiiDEN2ll_|83blKgbkKpLh5TT(0wcfHWll2_%YQkX?p47hh1#EE8pa340y(AT#I&XTa)bDAda*plXwTf%%iPGky$ zr$fgo{0ZrCZ^`G!+Y7Bmg%?at6@)i8<9|l~k)pCw{5K*CO)Ta3gB^ljS90c9<}ao@ z?(d%Lsum%|14(_;t!(%?1Ka4aiQtqm-1GM_s}+R`&(L@NG@Lt&2)c8sG0(AhT17AZ znh$OXfzKDhn+C!;q`s9aDeT$&fKu~c__n`d-|59o(SFnEIB|F!Ee5zOa&!dHE71Rz ztZE}q^}-m>DbY*y>A;w@3r`&*z>_@?=t!NAulYSew9`xLkd!tqJChde5!5*EMzr2$mhV9~~U=-7(&E zX7sOIj}!U&8*rb>RfG1@kVDAWv|m4O!bRo>c8Qt7`)sP zT9g14WXrfUbXA0N!WWsF!k!YXMK2HgaRYiwIYWF*=kN;(ZkT*E*Qq?G_g9;s_nX^f z%IvS+J4u>Og~)2NVZHXQ8oRi1#!j+d`aA)=VkDN=)LQR4uZ7GhJw*=raRkZsD6%9_ z-ya7_s)&XNeJDRvJvu9zG0m6*imb4^8XONkls2RjIyvedwG_&bGKx(^Q39ASf1C&s z&cHtPA7Lz*wc8B^*C#NGwOJg-hp?2IhDZ3#6EEqjhW32{Oa38M%;QUOpEYUIT+HpD zwjRL-xk05nmmKD`y)b5HHFa)-YwywZ$?mySY;~YO)36szFi$ju3m1+q5C46-@N>1L zkvAregV|UOE2oIY=_C0J=hMLr@{_m~6DxHs7lGSq2-508xck~%8ex*TOx6rb5hCSJ zOY#z;59WL!;KEuqBfhC{9((Fsa#y<^oBek*s7JQc-wpOHt0-`a)O>l4~9%Okj0nx}P>B=7Z;z6kB)> zWV~z+l>CM+B*j`y&^e8L(1qJo&^{Xf$>ex$kSh_Bj-UdmRQ*HAinK4}Pu`vfd>5J= zUAp2?>(_a-yAR8FIDefEu3sowSW?KkxFMJ@U1$EcpkUBHyIbm-n#n`w(fy6I!E;CS z6kl_Qc6^Atc5J9yG)l801yH8))4+kram0EZc(*F!@L0gn-%!cMvsaLEv@8*aA+iV_ zfLbbGr3Kq;!$%{)XAA1NDmIAId)leU_URaQc&J+Y@$9wm&1<7I2@u~*S^StT#r{tS zSC~&?kR`UQTO$A=p#E?ba6-2JuzV{WU$%$ecs?wa>h^Qr#yhk-G?n%JI+fHNriL{r zZ!*Cl!pZ&jmXr9l2QrE+H03S+qBit!+O4ync^2JxF} zz*qTdh_=80bk?46^mQvviRan|lEKn6;l}4&?&1QWf~79D!iWGSTdY~!5HJ*tuzMvV ze?jjC*hF`GZOUbSkMA*HlnIj?c*~(j>KOu=!MmCTv^@)sIlY*S>pYHL-AVork%KHD z*URrBtmuCLen5f0{z&8Q0JdXVAZfTNh*&vBK#(eR>c_4P8g zTlb`bOeV8j=JTTqE!^xXqCH2@Qhh&Bd8MCV_C5by${0lJRo658z03{Xvn>Xsjy<>S z#~BRxumv+=Uq>P;>);-U>}0qHV#7EI@|mSKIZwjxfgo{nDVb0!CB3x#NWXkP(xSqj z9JDt8@yrV(yDSaDG^dQ@>Pc@?FaN!VZv8Tv4!~n`=W80yZ|x)BR6lCF4??Az2|!M} zfxM2yK)=T|tDAB-sV0ZlB1rb(jo%2<)G%V8r>gyELgVZK+*Ne9Mxw%pwP8TW>t=}( z5B3dctvf26RN=+yxNsrhqZm**pbEfObqtLA>bf-kNBb{Ck>f?O1+g}=xG!vFpSli{ zd5*S|jp%DHGq-h+eca?A@-Pw0I+u$@{hvF^Y7aY!w68jg^tZc+sof;<*t+4c#WlDTR({+4hi_N97<+W1TBrQMLw0u9=$O?a1=-vQP zVqTzZ_|hOjRY1u5jQ6^R!?}M`zeU?RaU*9pVi7bb7RGz&DaonS$b@XXvQrLTwLe!c zZuvQO@`wVxhUG>of3_>sVV-)ujC72Y)3$68wgBSo0eq)i|winspkayf@Yz-LkUkM79t^BrwXXorc{@XB4eXK5S5rUD+<175epLi=kD)|>?( zuBUxz^k*sTbm=)Jx9%L4{X7qUS$JNtZDT(1ReS;QbY>y@gV!ZucR&#vm~@#P<6g{q zTbB^eo|X_rT}p|vl2YQ({j0=Lc^P{rznm@Ia*fS8c%41`HN$?t`UX2N_$E7RMg^;H zT}iBtt77kWeQ$Za@feub8{2=_yk7MqeYGCmPmK4)Th`>&ecn$%eg^S2wZxD0Fb zgdEoBiFL3>&mM#|dM@8bf@coo>h$bauujj$z&bta2J7^!0M_XVt1tLEJ<)MAU#BPT z+Bgt(FC5qoCSu}JnV7x(%#k>F*oi$?>dY2wb76nTc4ZHwxUtLDxwES$co2J%J=vK) zUc}OI5;n@xn;;}Utc#8>G1^>8MAk|PJ1sxLIp2>ks_-ZJw!~b_o`4W!gGu;%?U(qr z#gDb;&)UXds&!@oWRfrzL1&GH>udm6W4;>P{{wjz8D*1w7xH;(+kBDz{B<^`MfOX; zm`eez0OPL$SI)kX^&5^8BBK*XP2D-da;E}uoOF}jT6B~2oo>K>^(m_IooLyO9{l^; z-OXmJ^qG(EG`N@kw8VF;nev>8Haba6eZ4JA2fXY7v>)(wNcr3Y!82b0#FaVa|3!5_ zb$bB&NB;cU=lk1UQ%h@m)|je>#`U+I1#_DRNGlFOyT!qLD@S0w>z^2>sCq;{M81_X z3d-bddqd5EEE0fBX2!uB#G=>LzcqG5M!x)@`x4)^cQ+?*MYy_r-jt8L&Eiq@?;Z~N zb23^8WOD(?=tdlj?+<$H-}2wGZ}eT|^FHJITyHqU(wYa_9r_W6%fSTi*YEIo>~L>*%((7f1WG1q*F{e~!gp+@BK;=5q;{ z*R{Cvtc>eM*?JkXeut5hNu~W-^;_{hjxatYKW$k{+|QeE@*lvx4uN}SH7(F`1PCN{BC;o2+X5ZJc8QC!+v`@^~27m%C7z$ z^GTPO#Hjl(yVuWed%I#)^Q*<@6Rz(zzfv%-^TGUn8xQfaT0s55IAU+F@(Jt%9iH;x))yDIs6Rj752~rv?gMiE;jCM0oqm%pn}KI={sck2 zK7lsf5h!dZ_zRZ;&199nV%daZd}D1fhc7Wh&R<~{7nd>y|HIyMfJKokJsu}4Vi-Xs zssl(m11l)3k&z&ai4`NBVI>G+S`ie(onv0};Hl4mGAQVYpo@wu9_V_2EO`+@qOKxI zK~^!K82;{Fcp``4-g)o+@B9CqufLwAir$^BuI{Sps+#B$#r?`s`IB8`(4N#7x9EYd zj#fpGibrMjgzi;A0?TUOXI0ffZrvXG7NkB58dCqrSA4oQs8>#%?~!efgO(3f(EO7G z?X%R7w%=q#yG$~M&u4(OmVZ7LGvQz0GqP5FyidXUOEwxe@O;xiSplAPEvQw|J_U7c z41-BqhJyl=ml=a+*8?zWuDew5N<9rr%u7@3=$L_#>oTC954CnDvb|wOPWg&!6AG|v zBX0{9s1{OBdlxCzjVU7BuJoagF6_%}{+XbEaxo;re2nOM7RJOT=YIG)O%uY~%oLyd z)Rb^mGsnrB<}LP73kRirR3H1S{)+b1+s%4^TYErj8~SeWjQc@356VUG%)d3$cLaFG z8Q(#j_Vaez*766OZ>nHVf-=y*Zm0b~i*)c?vOu{Fepj)FQucpf{r~ndINsk<{x_h% zZe43@FH-=TSbGBb-%~kfMu9mq72KFZy1GnDqTHu`o(~RJ6+u~6kR>gl+~uVTEV2w+ z;eSWq6?+%kvJTmxDe;{9f{az=SncQs0)kzEEjO$bL}*q~;ons&3LC1i!txqH=lqA1 zM#>|F-Q`;B&C5DP@}b9+Wl_DLakT=Q{zgr(qmD&wFH=_}W_O~(O1cZmrfRnEla3-g z0FI{x_9x!w*?c}x{R#eRnai#DYz7#kS)gnJWg8f?Up$q2g8JVX8_;S`_)>hpOY-yk zK->6$aPa%Df#_d?N?9_hxRL|heP`n)Wf0u3hLpa)G|_Guz8H2#_69inpU3~FLeD8f)ZcAv3&oD z908w-M=t zN^5Pk#61W0kgg=m#m7p_r6Cy>Vxy}+aAi^bInzT2NLBU>6mO>nac!4bauVmWrGC7@ zVuihxwBE{E9B49xYusZf=Zvb2boz4}acPw;cSiw-gQpGSc8ed*IdFD_)c(jwv2@oc z?#i{JId7KONzcqC#g?vPq=W42#g_&VED zkuze=Bx$v;t9a>;Jn540lf@0A+_+%_rf|&qPLXl-e{gswg8S`4D>7GSEjrQQ9AB~rwU5o&i`vFKv~o*j zqk|Ft^uhZO{CnaT1mB^FyicUI9S$IPVgstB3FQY-+rHo*2p=q%2!CbVVFWimg;sv? znL|ere8#pA1TR~C47E!a9!G7D*`Wx(+js`y9~c#e;0^;$Ab47zlL)@DOE`kRd5>yo zQ+)(#Z@7=LHE}I-Pb2);B(%dwJuaL<@aphL1V8x8Sp+xO8in9#0nrHV>T?dYC(SsI z+P7Rp2!A6d2H~^%UqJ9!-HQl5qVpvLuX&AX&a%2#)LvSKvO#g4<-{TUu&ZK(Zx$1e z;Q66f5d6n|zacnlLjr>DS&@j^yFIU>w$}7(2>-gH1mQc|UPtg}Cch*28l5BrH&;tW z@N=(F&5f=}L2c{O8wg*Xo{I1nBuWvyM^qYuA32_m;Lf`<5IjpcC8e3mvQXR217+gn zVv}Mv_X&J(|n{x-@r?}p2Sx?~ue5*8!|M9E&{EgFWx8e-` zP`9_vxAybss$~je z-!QcjMQ^44rg*=rtVL!Lg#YfXj%%Z#r%Z)s2bXnVDsx14tUcwT(zyrmM*r2hg8AGJU^51qk0 z;erBXq6v>b z7R*NsbuE~W80uOuA2HOmU_N5OsEH^av7qljWLFY&=wicMtQk&6T^hlh4IfGG**}We zy?Hb(2(V+8d6Bf+)G^FtM|*n65C_J(pCdlGw^Q5v0e>YPVMrTa^tSSUYrdNZ#*hcf zCNQ4C&Eu&D#?qzP2IzI=ov5w-&!A0gbnE`YKp)|toCf`!S)jE4T`BwEH~D{2CnldQ znFMWsf3W|3r`qw4*B{&@KXpO(N#s8{cI(G<{#rX=TYWhe(5ng1t&5M+mNCcK{j}op z$ymOVSx?z&onG=FmNqZ?l{P=BMn@(u*5waY#`-|sg0Vi3w_vOfZK0X+}x|u$6AO83IBmdRBHh%|wPONT!-TYEMo1K6zIeWz2GHjVP?|vhZ+(>Os+sz{ zqwixh!~X8Pn*XjdMqZYaz3jfH1Mj?vqb%9Mi5J{)tn8Grvn=|tvrMSsBHNqoBJ(RB zCtH4QJa1d(1lg256Uhz}rhZsYzkHzbxl6S z0T0F_fE#$E!Tp=fLpzR@Uov)<|Mu8feo)0lek|KX9#lR~zW&^J{(;O1^7(rv@>fMo8vPmm z{G`R|qf4pNh`-Wy)tYw#0Ud%si32o|f*Kkzn8?a~^!D*bneul>Y4u|vbZFW!de8df zcwS^EEttp9s=LDI=VMOLJC~lsEsVqQYpxNrhH4~!_E{vJoED9fr_a%gnDe;NHW4n| z9z%~7T%g1KFVY0>676dhM?dH$rt{y6=^OR&_$_Gyo+3`9*M?rjPanF5cR4D-w+CIv z*ZBU9FPfP|8`>q~6FDjP#o;$->C9BRsVJ2m-yw}2`6LZre?J{x7Mp=dYBMljDigm} znu$+7kVW|&UDl zkIAb$>dEtK6eQ#KM3Uk8RFW{IK@#ooOmf`%xn!^L3kjw5QnFm7QR30ih+7=(g6r+< zN;_@tM%x77w3%0T#&oJCt?sDBu!i(tUK#gf9%}VsN>sF&%m!^yM!61iE>D*UP0}MW z67-3LC<7uoq&IPVS07^Uy1oRpm>`zVHY7YI7!gxO8WWBK`Vm77ObFA?rp)AyW{mxF zGse2YoavWu!Sqb|fo{CipRNfXKo{>HNM~#wL?;AT($QY*mOP~`Iwh^Id@oKOQlWus_RQKpjyivk^>c0O8;7U`G*?v0<57AsCk9*?}{fMY6x%nw9O{~;Ol|B z&(BiB1jT@Fycc5fMGG-}FQ{QWdS&mVOZAD< zFsFOvThMA>=B?KJ)DQbQeHZGt^plV0aqCZeV*>sxXqm^_U@YwXFx#}{XgZ>vOP2MGHmQ@^8&`nIGz86#C`grVyGy#hfCsIp7uVRM}UBliV zkzi|su49XRf5+y`Oro^xk}*3@3KlW^26bg-DpgmMN{#A}Mh$wBhAqFJj?ItFP{h<` zDBP(`ET%LQb2yMCa8Jroi2SputdwlbBrr!27?q>g=O&}hN60X>X}O9SyK)r)RyV2O zEjO_|_dG1w8od@4FOr9l+x@JvKoKNoE`K{g?naph;FPXccUotvCe`H7?z+Gv{`=?Yjz776{!82IpL~z1xuX63 z6V@%U7GJv##Q~mhScKs|i#2bX!s8o@5aVq0rnnsu=r7gpF z5Wyibrh0rX6TRgolioj%+3S6a$Q_o?EO#m(F6!Q9rkE5G2VNBtg098Hs-j{>yP||} zk1r*LWS234C+-lv&fR5ZY?Kp?yYDf>Jnj?4OUfC2`v*kA#0uuEVI@;&Q$^@$R5Mq4 z)evtU)-WM29ujxbA2B}{)Dn`&I);DwF>zu?J!5}RL2UMWLd;wLlwqbc5M$;&V>Vhp zCyec0FkV_OiH?06nXwIxMAf@jjCtN`My0xma7uUs=Y!+7*-z1W&VTfIIjFVn@H3## z22fG~t%{my1?54>+Yh-U+uBDZWgqVsD$Vj#4fy|h(9e@aO8IHzJjY)ovs{zdDmKZI zk(Md!ik>$}cb!yGhxe)Eh(@WX{7xF#yD(jpay5f|AD7AAV3S3@>#i$G$kQc~pXjl@ zBK3(=xd!a9J9>+L{Iw5hn}OJZvgw0XrP4hAH){YmJ$y)wkUk{U~>HkE74YWYjVE+ z5Rvb&p`=68P|-MDThgS&Ry5!hhgAD*nCyq5;qvt05j^MLN6Mp?kK$RJ7|l)Tc&sd|6IZ?@i7Pw*&_(Wga-1wUeY||| zrU{>vpFY+VznFppnC344=MON%OgYz)L}y0`9}3CxtYcC4{i9 zOw9z`)IM2%dC4dQ2a{r;cd%;)LVGt)TddB4wPp7(j5H^_pQv8#)C<5WUiMp^QrM|R~s zTV%!C(6t-yvWGQqzD9T65rGY_w8)0HTSp+Cn=jyfQ)w$M-rZCDV~L%($F^Q#QJOt( zR6uX>SHTYASu^^Go&5TWU5E7(TTJXPw(dMYjP)HT)~+AO>ulyIt}J)ty{a87&d(ge zyL)3OuVSAQFZ=i~-o=f>dHeP_^AhK|h|jDa!P`D+quno{w;`2+>JI*$5{$6gc@BVPqM*Zi)UK9_tm(p3jM;d*_2Gx%t%OqXl4a+$GGP zE(B(qiE&jphZ^d;R z#oWymbGT;qX{~CE@5$Kf{t}-Y;Hi%3^!TM z)n~AeImmV_X~orNyw83pZM{$Mo3=hd;iFaRz}e{dZ{CgJ#`dizx~ATU91%IAYjj=C z_04cN<`ur7Z)SMtIq!U}MY;I~3y&6Pjf=Zv@Kj&NpxqH21D{r=1}o!C4JKL(LQf|P zLJ#?l3Z341w1J*5-O zmvxA4Bf5|9Q-?-_PY`cl{*|`W#oaxr9VK>DqNtZrE|;U9_;q@lJQOiV;x_g7(irpl zD=?-TBAXQ$Zoh)V{CtG>g^1YY8)klkL5+)};liyC4Mj67jTbJ>H5NIyGF|8uXDYJ1 zYqro~v$;s~OlPn?$f71;R~G=L5~5tn5=usPt+~I*3NG*3t>(OkHN07ET@xp;fhUV> zYPRSIU}UnOW=W+j$lTpiBrdT75u#qA#cB3{^64$|43^i1i0rxA5E0JRhKQbVwIMa_ zxY`iWy*PPoNKFY>8&Y$Ks|~67k*f_6rE|3*q95kCf*OPyO`6&p?L|0M2iuv_$Tqbg>TeSe5>`R zEwJG%zNzg#{|LUF*8DyAKKGV>N^isWVg+C&4`fh+7K!A)yc!hJuHwK?Z1S(Mi=7!gNtzMbu!%0uUP8hRU(}e(`+r9 z&8y>XpWk|${JhbRdU82VWxkXh)#bVo4+Lp)x$e72u4cX%ZtKgP)k4|DMt~Q`BdK(4(Rb^SJgB_ROotdLC?j2vr?9q!Z}uC=wHiW_X;99;pGep>)XJ+h^K zIMox(F14cqBYFY1bbD&*cfCRP5C?44fCH~qQ{Q}E9SaV8+yBe z32vjXqt8ddW!xIH&qvSi`=9c4(*s=IlpWR0KjOb$haQaPJp|2v0vg%9m;ViV(&Gc< z$I7E6=U6G%eln08?jw0r`Evd=C4-Y?EA-BKg^mD=*a-5sXG zz~p;vG6vUyp-DRACa4QGGI~t(bv{^-)t1?i(GITPug}c?35WBxv}eWyb%5@RIx@CX z4WPiI6T=^D2=xR;q@b%Ye8Mv!^)yU@?lV*J$z3z>xY(R5LNR$^ss))E)dga!2r^`? zCA>GOE0es~3Z5I#joCTY8pd|&&iLEffb0evW|FP|?7J;s20gL`n@{y5y-V%DqKIDP zz;t^s?z`Tkafkz4u%HiF>(>{0xc4K=C-#RuZ3d91`VNF`wFfab%p76$14kyaZZIs) z8Nx)|915zEofzuGFd*A8oS7Zw4ANG(kZU%K0HNbYlC$Qzf>piUNLRN}F#h={=IdVW zf3xYIP#!U0Wa6dG%aeWASArw&*q7+1_h+ZJ7MO26AC8B!8z8mF3 z19>_2bJwEJkoa-9Qm${YxC&A5NPiV?w}K~}r{evr;JHju@uC$x<$4y&i&OBF>nAKP zQNdHLv9P>UzvdV}qTnfQ2+KR6;7Na0@v;;=YKMw2+;$m9aeBpX=k8hOm;3=n$eKh~}d=1|V9AC`dy+ zjyj>gwNQMf^1*(lHR7?Y6CT_1!=_&;3*#TV9?m}<^E2N%E`o2kD^fB-O7m-0MMFPc}HW<{X&pcpm;q z=0HEoTw$8q|C@Mj7WP5*j7ubZ$nm6H8HBtJB2z)L# zVU8=vw|0Y!>nM=adNe$1{q-NF8XV7{WbZ$xH)NQZ-Y8C7tR3~E)2k!L99)+q{5iWZ!=`cIT_5H zXBUL=_(j%6K+l9rjB#`!SQJu3Ms&O*)IEBKocUOX{M}d&J9tSCi#*AvCdIU+W=Q09 zRrt%rI9xrqJxrL`fm}CQ-gowMvKQ>VDtBeJaDU9}LVQ-vPKN4r;6vhI63ILa$vz=~ z>tmFO^h<`Q9ML^Qzao0h_7X?0F#|L%s@&9YS5kTK9nUKehSwJ^V-$d@~I zR4+euIg7@2RU_HeqR|QaxSR#{ObKNsJplz+{v=oPH|hc%PGW}}}in-|u`i}~v3 zMTzZq+4y!VF=ia1@hdT$T*;kxO?kNO2G;85#-XM#67C~uaN@ft*ug#;dVaGPt{o5q ztY+?m1FU0Vm`xmPWh*}u8}zT*55Inx0EJf%z!ABLFz!MUw2C_j#?#5L_2v|~WJ@aT z3DW2^zjWGOc!>DQ<1mdo9HCeBJxV{*KStkwb&S4Tok487a-4X2?F4ZzFOw)ec9Px_ zd5So^`!rz|mPPCmogu^v&k{jC*>rE`bHp^q^Te4UIrJ5uTw3QwE@3jZ8J46om%ome zaF?BO41Ml#lM*{vUd1w{J)Gaj|9nXP@xx4~Y9EuS7S`;cE_Qks-U1NDZXmjY@K(E$ z^IyD@l%wSH;TdFjbq4wUo#W(pMJJg3d6~?fV<(wy@u!$?cb{gKiL;oQqBG3+g=ZNT zpKPYjm~%{5$Ma0b9yyGbaV}Y$B=46l)BB<$DeGrI-LmRL6?QbFPHP%bDbJ0lsQboL zaH$EkKHC&a&NRd5M00FscxP;#!~*mGwhK0QDS=IzZi$Wd?1~L?vch`Ub;C@|tubAG zcWOYRez}jE_sVV7FMYLNU;M zMmIG!mlnjy^_nQzLrc$QMS<++7u4BaM)XrLd17mcG*&3zmxX5!=4|aglQ(RwwD*%6 z__PN%VSMyG^4Wy@pnUx=jCH?X;oX1-Di|hpUZv@rCyn*S1tb3`(J)( z|KTzB5;RXKh|k6lsP2E*P)5JD?7RV_KUmVRewcV-|JOb<@5p4L3XmPrC4b9(8M8<&_V-|?vc7a@9H&S8|COcIY z2Bx^a{kuHgtpwi~R4Pp}yNoBzEW?>QWzq}oSMjr_uYPVmsed)!m$h&I$@vcjdLOPU zo=Bc+5UoeD-5JQ~N-)xs^N85yT|3jehqc$c5=J`|h11!{*8LnFL7VQ1B#v*S>6fdb zi12yQbl&Q{^f9j(qRx3AL0ZQWIW}>Goozf_^IJS!iX{-8$`S}zd4O1&n@E&gNFu!A z4$?t%GU2c}g_Z@sJAU!$kRgv>uu3j|nR1hhUm8;1j9==%;o_H=3C-gd#^W+=X;DTb zdR-xU*j**I+pG3HJpPT__fVK$PQN&Ji+*&7+xJiq%Ftu{?h*k@?$Ndr@6%H6Uub^c zUx}n)4+w$TL)zP*g3zn0Abjg9i6=K65ntZ=jVL<)n66EKLZt4gq8F^KCjJ!(X^&a* zeGkO^vE06gwwBz!hiZ)5_fX7m`ySG>HTU+!#6bVzAO-}cp~?SI1QtO@ODJg5U5-KQh{ z%!k8xjw6Thcw=KZ#<$jR7>~QlVSLLO4&zIrIE;U>mcw}OS#pd^$BpGM-piK5xW2AH zj`63J9L7s8a2P+lpTqdBEzK}qu3J1M7*q8B{mY)MKjM$L{8qPGF1M`TXl%Scq-m_C z&URsq_WJBa`V@=EQG_u=MHoH{jsEBjbW-;)qV<2bPD#5PUX)JVbx~?joG%?3SRlnx zFX299q4b{g^>+ijZ(T9bu&9I<%qdmsptOFZ+zzw4NNYA+`Bb|2Pup)Tgl*P-2Yi(M zR%INy-_*S4Jtu7DHj2fg&m?VgD`wX5LXY~5y|8tqIpP2 ze(3kze{7So-x1jOgqbrhTDWfYUXt>PksWg0$JkiM%GTJ#kv)1O$m;JVFk4<65cbJV zWSrOL2|KpBK>jiIf{-e_NFL137wRl2AqPj73N>6WGi^P~WUB>NnChNaWixcHk-Idm z%bY8(vu833rs+8dCLu%XN9~5+udSbC<&QHTocBiwTobZUSWw%wTnnIp`DCq8L?~e^%Fv z1g_uqx8$Iu7g~?T?^wCx531bpGnyWFUZDrR_x@Ns;ovyw=>kvbtsP%TqY}OF<;%xQ z*Gq)bty3mQ=PjLx|C_xdfvYL~ew9|G&(fljXDcl#DJn^+WC__B{~1?8jO8jzW67nY z80K0BHFaBT*+SMVSwo@1{Ikx*3|Znbh*9}}-@WhVsrTxY%$WJld%x57diC9R-#O3GRMDm(f5;a&B~{Ob z#IWPs*TE;azQazE>RnH94lbv;HEw6fZNsw2U#?}5_A2K{$@6nu;NA1wfW6sl@{??~ zHZ9={|-ThO7S9$v%w`%L6+K7XGXyMoW($GS7l-{*B_({8-( zjPv)g?u_&Ic{{e`kzmK(KF`~+IDa48u{eKUvT_!mzt8L(&FAkk%f0#heI~ROpTExx zR^#*cf%R71FTKXZk{asgwW}{t-G4}xcBF^meSLIs?eune^;i7S`sjz?23jlcXA6DW zGSFJdLU0a2HiA3^#R%RYz=hKPwMXmT9{txhR5RaDw)WR1c)w?AxIV>ITCM%8XeHMC z!}z`=d*3U!W~=iN8O;G~a|Fi_oSXyTlQ}~C`0)0mNbKXrx;eY)Q5?DMgKkcqV)^(n z^P3PqhU10!G2K;&AA?jOehk+O@nbOgK`NP4uoVOd@nc+13Ga8fo)X^ga6Kiw-+i|D zu|v1YwnW;Lzg_$o^CzZ_+Mg3H7V4L%SLCvqpS7zu7qH*X1u$x^VDoXO55)slCV?}h z8EWGEM63&89;wJ1Vy!LA8@SdM_v9R3LyPO$;}mN;=@c=|^)xl2MHaX0RTgpkRW>(u zbq>~L$g9jnqFwSOazB+z=tty{>xg{fdEjMow&xYLyZJS4#KUXs1wAct#Y-*R!<(FT zMTa%ZsLz>h<$HK@rHj8<9w^e0i~m?vD{XLp)F~ z9I&_s1^LD$md&{u%VnRABf$Qj$g-Gt){Noh3b8#rk?i!JB*Mf$nRA}_(b|>WPq}>m ze7d~{AIm0-n_cC8wECH*9{-GE+1%a^Z`o7V-jexd@3{1X@5uC$5^l}9QZDzmGIGy? z_uTH&fLtA@LVgveO3v)2#?4q*k6US@PIfTSAY1Ebl7>cFq|RF;LU9eA&%A4PSKD0)k+A|8$aWfiHtGyb*5x$MVZtE|GKY}hFAu)dpt<`>})Kkcb z)ccq>l;H<6!~>xSW`+Z}EL`Yw;gTx#1CZZTj%Tat)LA?m+qhr4Z%3qI+w=Q|`0(;x z2#;%MR(dMCq&09|pn|6IX?3>ghjlHxzN_-(FoGNi?_AzJQb1!+{eqSX!nfUn-ccvhtruH({ z7q}5d=h;EXRvy1SyCN@g&tuB<2H+~}8DF~IP=4;*_w2X0&15Zf&T<7;$sQ5>Ne#?q zPc-D~4qAY{{AE1#b0|Wj-?YyB8imy?tsOKi;)C z{_W7ngohIg80<#bWL0%#I==t88cJ*J|4y2@!u-S z9z5dr<@G>MLp-ntff@y%F(t&riVzG%JaHZOro|KPS;i8xuf!64_A87lK+V8>TC(#o z^xymhc3E2l?V_K;=3&nu4CbK9u;=iF-wSxJ)9>(#lzqGYJ zvGvmsOwZ1rw*T{cd=nk{YqYvE<4%YcE(k&q%tW+VL@l_CA71GYLXS1f@oF!d_$Oz-sLmJpk==p#Egpg9d;nX{7kKX| z;=5bYbC|V>J@PWbjN)w$N&3nwk`UFa(8d3woCQd#2f+=T@cJdFb~m-Mx!&DU$*FAeA%+dgz)asOxc2L=JxXk+lio^ zX9LCr%FW9mW9T}u6>1$1KHR7L+&KKHnKKM?>Xx-dF@D$gV2er51W%D;a ziw(bWvIaN3z9#49tHlj{tVIsB)8SmRbx79+x>){l?616!hb+CIPnt}FLCb1Fm3^W;R}S^($5*2#%mB{0V{d=8$_aT{+x&q z&Y!a}A&!4$&mmEl#yF_l_el_{O6`Mcs=WX0rCrtQR#BXK1){@cM2}zVkRFO`n=k#W z0NK}j7{)=x^8y0<*DSx+(dGmGm7&9Hg#S(v0QQIgY%fg56sdn}X}wB51=c0uQk~hJ zc1tbfopH%MbAs;+k9((ijuU#PVgHuGy=v4g7kb7!S8>mxM{abd-7>n+UQr&jS!`Fj z)mTsH8tFx=ckc#GgS_bnZG2#9^=IJV_sis-fup|ZG!WCQy!P}d;mlia^t-y6LjRXQ zQQ>vtH_5jcwuFB)7=cfXr-cn0raL&hp#>ihfl(Peh_~ zt;l2fb<;pTjuo99tmyBN&40U!@TnS!^!P{sn?wpdg~Zt8m$GMzxLcKQbr2o{LkG_( zKR>42b}ZC~hw|}eF8#eKznCq;O@9PK5uO6kh}#I}v>P5@9=EjtiT+bUQ`o&}X9Ipb zQ%Qejv#0&Fx^o48yR1AC<&Tq~4@uADsVaT45u?w^NV#>dtd4vc;d}BzSO~`{i42a#04Y{lBjo6DNM&v5clo)xVDYrA{PxAwONm*stadTiFWd3v9daj1)|Zn zh*opai0e?u^P8~oiMd?qZ+Ug8-{4+~%AVsbuX&sHbb=_4r93JpWFd<}0Y`;RY9})k z?Pr+Bmm_@bN05PVcLt3Zy^4HM(1c8)NBAUH>67xEmnYWID9`DZ`l2{XA(#HI-XBFe z*pK{hRvkZFYbRgCEt8is{q^P3sH(k^MR;!-g;cyK0PUj$*&WclU)glO>?(WE3w@T0#_0bl@nF#x^Jwcfl=k^BPt)-&M9m_%aE+>P+|j7l;L9E!{( zSFGmiQjqTh1sR5O<>k}>!gVL8^yh`thC2_bHF=M~nkG-codG@P za~22O)1QOkU%de4t9}Qa-Cj~|b6S;e#Al+K_=xQ+$&r1tej>QdhtP%aN^ zc*3t5L5XRereqFJRWTLaMS#^uFnSSy-=WbG^G>qVxA7qL;~bu!kJM=nkGN*5vTYUZ zJ6b0NIf(1)O8p6P1J<(Z3Y%hYB+B6|S0aBsRiy{~fbh2;K?cI*nMHz+{RCQ9MF?I= z>yLp6M(6pyp#H_#VBm?Izj_}R?Of^+*BXye8ds*u$|pg;QZ8_W?Oncs4|@EeAs^4x z`JwM3yxm7oKN_%&qX9IJu3S6RDY|kET36|NamC>=X}9h>i1)9`-@ml>i??tMu~w>B zA61lz9_Sk0=retzKlEq(b<2m^^Y|g#pV4fzFkh^xD)wjWkLC|W*BLHzofHefe;_;m z=2ybJ;1M6sAC2aXL-WU@5&u6GIO@8*&lex!_YuEGD4I7D&Hqk1Kk|N1w7mHx9fWz} z9wE+;K=T%%`7zS@Yfsu2$(vv3BmeVPqIo}}`Rk&EUI}Z}=f!vysT(_l+%U#|_S_#X z_7tzj_fnA2{R*8wzNd=&Rx-NgR&?zhB5quBmMfwUaFYN14xzcn&^3-n|MTng|C#mq zvk^YkVgPIuQ@I9nix{CEIqkMFp3=-npv|Hf>f!oCS~n&e7>~+_)8G67h84bpLkmiP z+u2g+dY}xLZGI0;*W#eD3M`$eN|{VmqnnPZN0kJq)9*YrsM`*jwDVF;DidnMUE8&( zBurXim2XBpKHiLew9A~jkk}kHn<1fg&Tj$V4{ix^CbWWg zoLhq(z1qN&1{Q#}wxoY4vjibJR`l`PR$$O0YdZO;4MkqIr5DDxqXz7;qsN8YQ%)-# zVC1;=lwqhN9P8JCdV}=+9+uA3b)@fGYjy&~rY=zTfeX0y&J{jA;|4Nrbf$IE+`&&7 zUFau2cz}7aUFnNcJgL}7FFLJXH)?K>H@(8%hZ^bQ3s3*<3+8vyO1GV=##oN3N9*)gXSBUE=qC=EjQ3Ja`Xbbp9NVrN_mP}EZC8Tt$m`k^oQqZzH6iR1FC}c_C*sob}6NbeDEi74E4{g6z2Y-(1j_VqfGGLS|gFqs?=KU+L@ox6Pf(X@Y~SW z!j$2!sP&;pvjQ>QNprb_0+Wh4A}++Yb<;}Ng((?_xAUw^2}PHblsf#vTz$<`*VJ%5 zoV;>3RA|iMF_fixF;T~2Jo?(ZnD)dnRBFsF4%591?C*U?qHO)!3kU9_%xYa6EigY) z!PK~U75xXjuXPYN$bE43yX+uO%0(gUeFh?sa}m!buli74>g(NcnZoje@3Y-yUs9r@ z#jiqiOA1T^^sj~%l6-GQlCDesNHkW=!rWTTmjpgmKLqj&@fZocE$%o*h1a}HDI{Uy zRH3-umsyJ-z0&Sv+7hW00nm}3(4(jkW^>?=Y-@Gpq%`tw&rX(0yF)c)J`t0 zKK!76y@zzh!R3<{&u3HcZng|GFpwu;F{q#-NF=yon7-P!4%a4@I!F7f3XS6)%PJI3^yhNafksloBjPZ2EN(f-okqkNy< z|45%!S>b!Hp#2sr1heJIooXei&+;-kEe7qmap?-&7-O23?X{FOEmBGy zZ9$+AFMAJT+nSK3V>5!3f$DroFIP@Pzd>n2og}XDziZ)v2~b^v^U;jf-|fOnGs`rh z0nn}YhH&#ITHd_AAwh^@szcP&Igbnl93hT^`UOb>`t~N$ z8iwux^;Mfwq*Z3+u>&PLx>cgR^a&zT${vV`&4Dh1_w}>w?-SbBokd%{*Ed*$DmIHm z;HTNb-cTj0{M7ZR44oip8-_hE%oSo65)0!zG_YZ-@H%qEg zqS3Um%&A1`g7+sgS6Fr?5;Ip1CULDdAHK!zdBtvN&<&$hs<}&$XAi-5bwG*iH$L>c z#%c3skf;J0Z7;El<#H$UyR0+9FHLeGxP3l6Dkq~29AtB!&oY#ZV(IZq`xu)e;(WA; zH*w%83h?MO1w*r%?Ovy3jV&e3Re=T#=5;gjyI5j;Y5&@=J=imx)7|e>P3tLC8&izc zzPHE5cVz2gO*ix&*PL0~r(I2uyUOvKyL~HyzM&yk@wv_Ay~0VN&9MeZ13F;Qx0G{a z3bdmt{v6Z!FNSl&>f5W4?Y9J++bAScO)mA;Z^&||e5!KHC0?JaqtmT?f7H`Xmw+>^ zf~dNE&3#zmwZFUAnKL1+Y2o^Xm$`>hoRVD$!?H==L$e=EZ7*kN%T-I4DwHB_OXIbs zpSeXB_4TWq{PIcR6Jo%q14l;4u7IzD&OK^s6zS|HZ(jC5o23;4rp%Vun3vBeXTD>l zTVW3r?#40Q#%&xjU|U3eJ{~E-|Me9n&63-i?V{O&??Dl$h$&BcF|Vzu^P5Yv7y+%- zVaw-S7@AT9`f)uu@^p~J$#7MrH5*eC(B6EPxW(=-@cYRnV<63)GjwuC`J7As&&v^W z)rYma36w`NRgL14S-{V1=ZS#)n@RjRTf)0J{U-WM11Hb%tv|@vL?H+8$Ui*w(;nCJ z=KQfV&!~s}yPXd&p;K{N69tZW;={$P-QpjYK2Mhqeq!?{ub`O4^+NnPvOD+tU^-nh zV)t)-KR^i@B>Q_%COU31{+C!|NsBo$yr~%w#KzhNMoI9MSlSO~R|P<1t6g+nKmroV z&*9_W)%D^8x#b(w)?Z04+uMtbq2W@8^)+Six?A+iS0aCY1==5!1U3`Nm`4LobjlUH zZs*&BI;Yl;!HZ^r4JV~8rNYFM=LEtvpD1oE7CPq@ntQgH^7#d&Yuqzl&g7$g%|cGA zf#fm77k>{qGr;5ZS!|8E%X_CY*EQu6#+4SesR8per_ShHNigGdn4}0ikfZ|U@LIeC|(a^g|pu@FvLHk`6L&8bfn0R&z;kB$-V~8aK#=zol?yGqp8dsH;mCpv| z^fRNNkJ<4S{@Y*t@=}!FGnOm#oyBM6P58adx8Rbd+p~0cnuu1-?-Sc#s%GeXvKctid6o9B}7w)f*RpoumOEn z*e8GIyrbV6cD$y`4)k9x&bpumojIbA0voxTFkIRod-Lg0wnSb@p$0qa3>_h1&(9|vXHFy?Fzd3F^ChLGh!Rl>qU+V7dXV_cr)WgS@5!3Ui= zf7suj%je{wotj$G2zR&SQJ)#QZ+=iUo|Ft6<<>fwqIEuJUb;aBzN|(?DKl-X5-}Nk z?(xvShKZ$q((|N7rEL|VZ!!BHzJnMOl#6ZGuhK7W-!o5INmXF)o!Q-Az6dOoid}Yv ziR7I{oC7b$mG|$Z?jKvb6vBc%ZyE1_53N~eRG99JBwIMI|CJ=-m|ar<%St%>a4b~+ zvYUuD{CsjI`8Vo1L7ud&j85Du$CvCs z2I>M!G($q}lx7x7jHyGfQ`Ex@mKYzJe881Ee(ZmF?=$7-e6nboVc_#~f0;|Sz#^3UAd{Zs*n9HK`*5TSxWK{T zKHk;(^s&j~d$ETif;^jEfZ4gj2yxQZor8JgEkT$xG575lh)&~{uf=YVbKvzxR$vpi> zMt<_?=QOiPv|-n~PiH}AS7#p0RQ>1}a}2I?&`Kx@rylC4?Mx=5#?2vPiGfMF-e5Xg z!HDZ@^zzM+8?LNC&lh~du#OmmQK?DKT(`M)hR~|J7qY|ZGG&ri?@MQN1VJ zKN1|T)40w6oj!@tX*%HL$P83asTPYGf1*s_?@uf5KTk^uckIqSbhQqD>^wXccSjcQ z;X+fT`n|BvDEb{uip-D4=LmCG%kwK&&GF|FRTQe$F7Q&yEXYhL_^16t5BreFE(WfZ zs&gq!m>?z1X+xeaZ%nn@agukobIq8)Lgop)sW;Dyc5V!x<{r&%qI_4-%wnh48r>jQ z&NrA-@X{i`ux6+r@H$;{@ZFZqp_<1oe5MSKB4s94@s#k_Fe$kM*{8R@>zNTtcZoa; zCc42c!%fWT@a7*{rWAVETZ$jGBdj_88sI_q)bYp-ArVB)K1rx`KlXcnl$3+)0@M#J zorxMl0;L9Se(|~vGBw=h^mgZ}VkLlixEHLObV+co_yC?TDlmN8`n~KI*mx5E=zeL8%(Ru# za1ScOx%`-4AbI|FAY=Unq}PoqK&)9<9vN9|G3Bw(2&ck@+BdM1MPsB zqDs({gTdE;lQ2vYKF|u%&ZL|u=0N*Js?3=Zj#)T2uT4k+5RY#@rN_9nrMi@c`Z(hh=KXfR8jx*Qf0=I&S(~HJ)5-J%#&7!a&g8lnnUOS}va?&>vEOICRgIb?!xrGj{F{PeKg}ff z@N4_te4}|EkPufl#NrPfDe!9milF3BddNJ4;D1I zUH*rSgx_vy4Te~v$l)UB*EoCC{#%}pYOy3-9F-q_UeY9g*x-|7=FuE)k`V|?f98{w z+5}^@4J2kG*p6trPgK@VQ-ndz5ZYJ*jr^GvfgCl$ke=2fBkm>#!*p0HQZgQWt`{g)5iQXS8??}?=0?i^6R_X+qTXjq(?ex+%$i*k2 z5P{PijikMU-|8siJpx(Qb5-JBVw)q>$GTpZtWI$Uuq-Y_7O#30C~y`uiWaYS2#WH^?<gz|c@B##>S(tRoyB!4`%a8Yb)?M3M`S^T({{`v9k?83}6yxJh%dn^W zrjq*s>+mPFvWEKi!kL~=L*=n}H67cp1*X_z>1tRaZ>-wc8=9f~6D}L}4LyRNi-pw` z55?0N=l1e`?bX?L&G*{rTD6OV2-fBY93Sphc8ilXX1PuH%~g0^%bsS(u#_e z>Bek4r2U$5f_(+gC^>bZWc|<`YPbEQ@3MHUekGOV(F0d=IO@?N_A?X<{3E4ia9IudYe6lp#Q;GA1&OZY9b~7&(cqgQlDx$VyhgKx~5wc?AOSPALkLG1L zTTb)5EWGc8M7R2Yw=6R-oUKYMiU3EM%+8Otv-mKYEY^fibWQW=>Kz+}>maB-e7k40 zJM!fF+c&*vs+6`jr(*rwo@zE5j&ZYN1X_Gjhd&W?Oxgl-m~V|r%Ze3XM0UK>s!TN{ z6K)f%c9E>u+sEtQJ=fI6A?XkmRT_eZ;M_%S{-iqdj(39s$+-kP)ygb({EbTHR#L-N z$x}UG7DN56g`=8$^cMyIm1Q#Z%vJcV12p=OljJyNbU5d*R=qiR^r+ z0rPgBy82@a!6UBJ;mY-D@CftQ^Gt~kXIq^;{^u{}v+9+#-v`OJwR{ThSp0fj0tXFm z2j$$?62%>8!=*!d1c>VU3UKPJIg@Dl2QOUygyl+zIA`KW*sN)}9J^`5*MrT?BDmij zNI+f7E}KP6^^Q8pDR0;2BthIADA+&a^AW{<>FLkgXe+ z8%58Pr?i!wi1n9xtJxI(QC(x<5$zs0_$l@+wh#fwd|NMU{ z?A!A5^q?ga~1XVV5-<;5J^{R?Zs)ORsNq_VDQ=E2+IL*EqQ`|P5 z7`QkNI(ehFom`d%X}tCAxkXOrgd};+8k}5C2_;XzstLc^U^&`~WffRThmDP5SCtF6 zu13J?Y>D3xAb7SEtJG+*C{3}n1d{&ptFaKTW9a*BJrkReUw_9L#;2WtG zLvc{oktv-!8HH32ryVHv-U|gQzBvT$&TSlVl8|C~)1{F(CO8u=6{>?o=ku`gIabt_vi zsChPJE5LPOJEOHzy({N&MB^HaoJ3?Dgxdrq#~x^J@gV4zjz_COB}>~o%hjN-4}<8r zFlaZRFpR{VKJOY@5Y%ynFZh7Er(7SqE;>aX7Ry|W@E3)mdGVDLmMyy;fYJ8GWD2F2 z8*bXBmaqQ}kgVPWKkc}k)KkFODNR%*aV=?~*c=$aT|NqRvtSDMn!Nfr9(i@`nXslg z)z#@`3uZ?Gd%Pg`SgT7H1LF%kUy?eI4Iu{wBR-sF5twGpf~lT}9z6(Zkt7qB279QH zDL@HpO%eC|F|v)!@~N5j0T2Sr@PBoH5s!J{!bZD?BSS z{z7pUIE%M{vxF(HK|)7thSPX4OD zejAR15#8o&xg!(q5z_xbPr@ba;A3THjtfE8@fW zN8stIAcG9;jFV^wMcdx0p47(}_BPHH*7_x0j8$lxsf_S5tCz|A_^YMC(=c4h-}^`S zt6aIVKEZ^a!bgEoe7H#f5KK{&q7NgmBE;7c(+$Xm$<7Oxfh|swST9;MBvOP~iq(^rhftVe89?TrVCQ(R^~D9KXe)aU1V_G z$Scyqjwi&%o25NSwVt4c@SSbGun6xg6QE|pVmDC5i5rzisq2M6C_8!=t-#1PtkU1!}BkI36ssU@y8 zf*lj?{!Y-p^RuV1$6dzc#>n}h8|*tu>t)6sgZ&hX)>HR(Q6lDuFi)AfjRNStRR7c+ zO|nGDk7a|^h7l36E+ZL(m4~t0#>_xaU!e69r;4PWKoT$zyd4bC$D|J~5583ThnG$M z&m}kB@HOcUa3<%VFtMKj+Y-%Ep}NA-C3Hi0@Qm$q#mI;7B*09tLp@zH_A0OCq2UYU za{g`7Z`)qbYbyLxpyd^72UmJqB10L4O`CA|yeJdRQyW>cv%`PhZQqsZL1wohHEyv- z%b2H}D@XX=fCWk~;^Y?0tti@YDXE1vbnu((-xB9ZA3Ydc|EhbDPdp~#aAP(|v}^gO z?c&R(`o3~HWS_b!Xb5^K@GMX}_&Ky;2WLw-wr84djS}c6mH!t(l}G;w}JO8u#JJH83K0UIzQ$B;`5D4#67$9Mdo+NK}RS828@^rYTO3T$5N| zR`66C_X>#{g${5km}_C8gMJRrNX|%O9k7}oOl^rj!^i+D>MZ|tBoeIK#%qb&^@9Y8 zbC>xcy9Gc6xX16qY}W=(ge*0Z(qVnXk-ZP;m5rk(zW|$ki`@VAymt5~2F|2o-PrZl zn=1~*qyNLGSHa6qLN!ysa_ge6W!DDzg6 zsMK6{=75CmQIyf6+W*3eS^xk(_AWNAJ|6!q7We-x7WH`6QU)GNX@IL?-^N3sk literal 0 HcmV?d00001 diff --git a/tzdata/update.sh b/tzdata/update.sh new file mode 100755 index 0000000000..ffa1e19ede --- /dev/null +++ b/tzdata/update.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# Change version.txt before run this script. + +THIS_DIR=`readlink -f $0` +THIS_DIR=`dirname $THIS_DIR` + +TMP_DIR=`mktemp -d` +VERSION=`cat $THIS_DIR/version.txt` +BASE_URL=https://github.com/unicode-org/icu-data/raw/master/tzdata/icunew/$VERSION/44 + +echo Downloading and updating little-endian files... +mkdir $TMP_DIR/le +cd $TMP_DIR/le +curl -OLs $BASE_URL/le/metaZones.res +curl -OLs $BASE_URL/le/timezoneTypes.res +curl -OLs $BASE_URL/le/windowsZones.res +curl -OLs $BASE_URL/le/zoneinfo64.res +rm $THIS_DIR/le.zip +zip $THIS_DIR/le.zip *.res + +echo Downloading and updating big-endian files... +mkdir $TMP_DIR/be +cd $TMP_DIR/be +curl -OLs $BASE_URL/be/metaZones.res +curl -OLs $BASE_URL/be/timezoneTypes.res +curl -OLs $BASE_URL/be/windowsZones.res +curl -OLs $BASE_URL/be/zoneinfo64.res +rm $THIS_DIR/be.zip +zip $THIS_DIR/be.zip *.res + +rm -r $TMP_DIR diff --git a/tzdata/version.txt b/tzdata/version.txt new file mode 100644 index 0000000000..db18f8311d --- /dev/null +++ b/tzdata/version.txt @@ -0,0 +1 @@ +2019c From 5e525588bfe060af1a65d0961764dcb0c7dc87d0 Mon Sep 17 00:00:00 2001 From: Dimitry Sibiryakov Date: Tue, 14 Jan 2020 17:49:43 +0100 Subject: [PATCH 216/274] Use static_assert to check ODS layout (#234) --- builds/posix/Makefile.in | 22 +---- src/jrd/ods.h | 206 ++++++++++++++++++++++++++++++++++++--- src/misc/ods.awk | 112 --------------------- src/misc/ods.txt | 185 ----------------------------------- 4 files changed, 192 insertions(+), 333 deletions(-) delete mode 100644 src/misc/ods.awk delete mode 100644 src/misc/ods.txt diff --git a/builds/posix/Makefile.in b/builds/posix/Makefile.in index c16801ea95..1d97d66a6b 100644 --- a/builds/posix/Makefile.in +++ b/builds/posix/Makefile.in @@ -226,7 +226,7 @@ $(TOMCRYPT_LIB): $(TOM_Objs) # main build target for both debug and release builds # -.PHONY: cross1 cross2 boot yvalve engine fbintl gpre utilities plugins rest codes ids examples cross_rest preliminaryCheck +.PHONY: cross1 cross2 boot yvalve engine fbintl gpre utilities plugins rest codes ids examples cross_rest master_process: ln -sf $(SRC_ROOT)/include/gen/autoconfig.auto $(SRC_ROOT)/include/gen/autoconfig.h @@ -234,7 +234,6 @@ master_process: $(MAKE) export_lists $(MAKE) external $(MAKE) updateCloopInterfaces - $(MAKE) preliminaryCheck $(MAKE) boot $(MAKE) yvalve ifeq ($(IsDeveloper), Y) @@ -306,25 +305,6 @@ cross2: $(MAKE) cross_rest -#___________________________________________________________________________ -# preliminary checks - make sure platform is OK to build FB -# - -STD_SIZES:=$(SRC_ROOT)/misc/ods.txt -RUN_SIZES:=$(GEN_ROOT)/ods.txt -ODS_H:=$(SRC_ROOT)/jrd/ods.h -ODS_AWK:=$(SRC_ROOT)/misc/ods.awk -ODS_TEST_CPP:=$(GEN_ROOT)/odstest.cpp -ODS_TEST:=$(GEN_ROOT)/odstest$(EXEC_EXT) - -preliminaryCheck: $(STD_SIZES) $(RUN_SIZES) - diff -u $^ - -$(RUN_SIZES): $(ODS_H) $(ODS_AWK) - awk -f $(ODS_AWK) <$(ODS_H) >$(ODS_TEST_CPP) - $(CXX) -o $(ODS_TEST) $(WCXXFLAGS) $(ODS_TEST_CPP) - $(ODS_TEST) >$(RUN_SIZES) - #___________________________________________________________________________ # static library - various common code, used in different FB projects # diff --git a/src/jrd/ods.h b/src/jrd/ods.h index 9b3d432906..73fd2ae28b 100644 --- a/src/jrd/ods.h +++ b/src/jrd/ods.h @@ -33,10 +33,8 @@ #ifndef JRD_ODS_H #define JRD_ODS_H -#ifndef ODS_TESTING #include "../jrd/RecordNumber.h" #include "../common/classes/fb_string.h" -#endif //ODS_TESTING // This macro enables the ability of the engine to connect to databases // from ODS 8 up to the latest. If this macro is undefined, the engine @@ -234,6 +232,14 @@ struct pag ULONG pag_pageno; // for validation }; +static_assert(sizeof(struct pag) == 16, "struct pag size mismatch"); +static_assert(offsetof(struct pag, pag_type) == 0, "pag_type offset mismatch"); +static_assert(offsetof(struct pag, pag_flags) == 1, "pag_flags offset mismatch"); +static_assert(offsetof(struct pag, pag_reserved) == 2, "pag_reserved offset mismatch"); +static_assert(offsetof(struct pag, pag_generation) == 4, "pag_generation offset mismatch"); +static_assert(offsetof(struct pag, pag_scn) == 8, "pag_scn offset mismatch"); +static_assert(offsetof(struct pag, pag_pageno) == 12, "pag_pageno offset mismatch"); + typedef pag* PAG; @@ -249,6 +255,14 @@ struct blob_page ULONG blp_page[1]; // Page number if level 1 }; +static_assert(sizeof(struct blob_page) == 32, "struct blob_page size mismatch"); +static_assert(offsetof(struct blob_page, blp_header) == 0, "blp_header offset mismatch"); +static_assert(offsetof(struct blob_page, blp_lead_page) == 16, "blp_lead_page offset mismatch"); +static_assert(offsetof(struct blob_page, blp_sequence) == 20, "blp_sequence offset mismatch"); +static_assert(offsetof(struct blob_page, blp_length) == 24, "blp_length offset mismatch"); +static_assert(offsetof(struct blob_page, blp_pad) == 26, "blp_pag offset mismatch"); +static_assert(offsetof(struct blob_page, blp_page) == 28, "blp_page offset mismatch"); + #define BLP_SIZE static_cast(offsetof(Ods::blob_page, blp_page[0])) // pag_flags @@ -272,6 +286,20 @@ struct btree_page UCHAR btr_nodes[1]; }; +static_assert(sizeof(struct btree_page) == 40, "struct btree_page size mismatch"); +static_assert(offsetof(struct btree_page, btr_header) == 0, "btr_header offset mismatch"); +static_assert(offsetof(struct btree_page, btr_sibling) == 16, "btr_sibling offset mismatch"); +static_assert(offsetof(struct btree_page, btr_left_sibling) == 20, "btr_left_sibling offset mismatch"); +static_assert(offsetof(struct btree_page, btr_prefix_total) == 24, "btr_prefix_total offset mismatch"); +static_assert(offsetof(struct btree_page, btr_relation) == 28, "btr_relation offset mismatch"); +static_assert(offsetof(struct btree_page, btr_length) == 30, "btr_length offset mismatch"); +static_assert(offsetof(struct btree_page, btr_id) == 32, "btr_id offset mismatch"); +static_assert(offsetof(struct btree_page, btr_level) == 33, "btr_level offset mismatch"); +static_assert(offsetof(struct btree_page, btr_jump_interval) == 34, "btr_jump_interval offset mismatch"); +static_assert(offsetof(struct btree_page, btr_jump_size) == 36, "btr_jump_size offset mismatch"); +static_assert(offsetof(struct btree_page, btr_jump_count) == 38, "btr_jump_count offset mismatch"); +static_assert(offsetof(struct btree_page, btr_nodes) == 39, "btr_nodes offset mismatch"); + // NS 2014-07-17: You can define this thing as "const FB_SIZE_t ...", and it works // for standards-conforming compilers (recent GCC and MSVC will do) // But older versions might have a problem, so I leave #define in place for now @@ -298,6 +326,17 @@ struct data_page } dpg_rpt[1]; }; +static_assert(sizeof(struct data_page) == 28, "struct data_page size mismatch"); +static_assert(offsetof(struct data_page, dpg_header) == 0, "dpg_header offset mismatch"); +static_assert(offsetof(struct data_page, dpg_sequence) == 16, "gpg_sequence offset mismatch"); +static_assert(offsetof(struct data_page, dpg_relation) == 20, "dpg_relation offset mismatch"); +static_assert(offsetof(struct data_page, dpg_count) == 22, "dpg_count offset mismatch"); +static_assert(offsetof(struct data_page, dpg_rpt) == 24, "dpg_rpt offset mismatch"); + +static_assert(sizeof(struct data_page::dpg_repeat) == 4, "struct dpg_repeat size mismatch"); +static_assert(offsetof(struct data_page::dpg_repeat, dpg_offset) == 0, "dpg_offset offset mismatch"); +static_assert(offsetof(struct data_page::dpg_repeat, dpg_length) == 2, "dpg_length offset mismatch"); + #define DPG_SIZE (sizeof (Ods::data_page) - sizeof (Ods::data_page::dpg_repeat)) // pag_flags @@ -318,9 +357,8 @@ struct index_root_page USHORT irt_count; // Number of indices struct irt_repeat { -#ifndef ODS_TESTING private: -#endif //ODS_TESTING + friend struct index_root_page; // to allow offset check for private members ULONG irt_root; // page number of index root if irt_in_progress is NOT set, or // highest 32 bit of transaction if irt_in_progress is set ULONG irt_transaction; // transaction in progress (lowest 32 bits) @@ -329,7 +367,6 @@ struct index_root_page UCHAR irt_keys; // number of keys in index UCHAR irt_flags; -#ifndef ODS_TESTING ULONG getRoot() const; void setRoot(ULONG root_page); @@ -337,10 +374,23 @@ struct index_root_page void setTransaction(TraNumber traNumber); bool isUsed() const; -#endif //ODS_TESTING + } irt_rpt[1]; + + static_assert(sizeof(struct irt_repeat) == 12, "struct irt_repeat size mismatch"); + static_assert(offsetof(struct irt_repeat, irt_root) == 0, "irt_root offset mismatch"); + static_assert(offsetof(struct irt_repeat, irt_transaction) == 4, "irt_transaction offset mismatch"); + static_assert(offsetof(struct irt_repeat, irt_desc) == 8, "irt_desc offset mismatch"); + static_assert(offsetof(struct irt_repeat, irt_keys) == 10, "irt_keys offset mismatch"); + static_assert(offsetof(struct irt_repeat, irt_flags) == 11, "irt_flags offset mismatch"); }; +static_assert(sizeof(struct index_root_page) == 32, "struct index_root_page size mismatch"); +static_assert(offsetof(struct index_root_page, irt_header) == 0, "irt_header offset mismatch"); +static_assert(offsetof(struct index_root_page, irt_relation) == 16, "irt_relation offset mismatch"); +static_assert(offsetof(struct index_root_page, irt_count) == 18, "irt_count offset mismatch"); +static_assert(offsetof(struct index_root_page, irt_rpt) == 20, "irt_rpt offset mismatch"); + // key descriptor struct irtd @@ -350,6 +400,11 @@ struct irtd float irtd_selectivity; }; +static_assert(sizeof(struct irtd) == 8, "struct irtd size mismatch"); +static_assert(offsetof(struct irtd, irtd_field) == 0, "irtd_field offset mismatch"); +static_assert(offsetof(struct irtd, irtd_itype) == 2, "irtd_itype offset mismatch"); +static_assert(offsetof(struct irtd, irtd_selectivity) == 4, "irtd_selectivity offset mismatch"); + // irt_flags, must match the idx_flags (see btr.h) const USHORT irt_unique = 1; const USHORT irt_descending = 2; @@ -358,7 +413,6 @@ const USHORT irt_foreign = 8; const USHORT irt_primary = 16; const USHORT irt_expression = 32; -#ifndef ODS_TESTING inline ULONG index_root_page::irt_repeat::getRoot() const { return (irt_flags & irt_in_progress) ? 0 : irt_root; @@ -386,7 +440,6 @@ inline bool index_root_page::irt_repeat::isUsed() const { return (irt_flags & irt_in_progress) || (irt_root != 0); } -#endif //ODS_TESTING const int STUFF_COUNT = 4; @@ -427,6 +480,35 @@ struct header_page UCHAR hdr_data[1]; // Misc data }; +static_assert(sizeof(struct header_page) == 132, "struct header_page size mismatch"); +static_assert(offsetof(struct header_page, hdr_header) == 0, "hdr_header offset mismatch"); +static_assert(offsetof(struct header_page, hdr_page_size) == 16, "hdr_page_size offset mismatch"); +static_assert(offsetof(struct header_page, hdr_ods_version) == 18, "hdr_ods_version offset mismatch"); +static_assert(offsetof(struct header_page, hdr_PAGES) == 20, "hdr_PAGES offset mismatch"); +static_assert(offsetof(struct header_page, hdr_next_page) == 24, "hdr_next_page offset mismatch"); +static_assert(offsetof(struct header_page, hdr_oldest_transaction) == 28, "hdr_oldest_transaction offset mismatch"); +static_assert(offsetof(struct header_page, hdr_oldest_active) == 32, "hdr_oldest_active offset mismatch"); +static_assert(offsetof(struct header_page, hdr_next_transaction) == 36, "hdr_next_transaction offset mismatch"); +static_assert(offsetof(struct header_page, hdr_sequence) == 40, "hdr_sequence offset mismatch"); +static_assert(offsetof(struct header_page, hdr_flags) == 42, "hdr_flags offset mismatch"); +static_assert(offsetof(struct header_page, hdr_creation_date) == 44, "hdr_creation_date offset mismatch"); +static_assert(offsetof(struct header_page, hdr_attachment_id) == 52, "hdr_attachment_id offset mismatch"); +static_assert(offsetof(struct header_page, hdr_shadow_count) == 56, "hdr_shadow_count offset mismatch"); +static_assert(offsetof(struct header_page, hdr_cpu) == 60, "hdr_cpu offset mismatch"); +static_assert(offsetof(struct header_page, hdr_os) == 61, "hdr_os offset mismatch"); +static_assert(offsetof(struct header_page, hdr_cc) == 62, "hdr_cc offset mismatch"); +static_assert(offsetof(struct header_page, hdr_compatibility_flags) == 63, "hdr_compatibility_flags offset mismatch"); +static_assert(offsetof(struct header_page, hdr_ods_minor) == 64, "hdr_ods_minor offset mismatch"); +static_assert(offsetof(struct header_page, hdr_end) == 66, "hdr_end offset mismatch"); +static_assert(offsetof(struct header_page, hdr_page_buffers) == 68, "hdr_page_buffers offset mismatch"); +static_assert(offsetof(struct header_page, hdr_oldest_snapshot) == 72, "hdr_oldest_snapshot offset mismatch"); +static_assert(offsetof(struct header_page, hdr_backup_pages) == 76, "hdr_backup_pages offset mismatch"); +static_assert(offsetof(struct header_page, hdr_crypt_page) == 80, "hdr_crypt_page offset mismatch"); +static_assert(offsetof(struct header_page, hdr_crypt_plugin) == 84, "hdr_crypt_plugin offset mismatch"); +static_assert(offsetof(struct header_page, hdr_att_high) == 116, "hdr_att_high offset mismatch"); +static_assert(offsetof(struct header_page, hdr_tra_high) == 120, "hdr_tra_high offset mismatch"); +static_assert(offsetof(struct header_page, hdr_data) == 128, "hdr_data offset mismatch"); + #define HDR_SIZE static_cast(offsetof(Ods::header_page, hdr_data[0])) // Header page clumplets @@ -492,6 +574,13 @@ struct page_inv_page UCHAR pip_bits[1]; }; +static_assert(sizeof(struct page_inv_page) == 32, "struct page_inv_page size mismatch"); +static_assert(offsetof(struct page_inv_page, pip_header) == 0, "pip_header offset mismatch"); +static_assert(offsetof(struct page_inv_page, pip_min) == 16, "pip_min offset mismatch"); +static_assert(offsetof(struct page_inv_page, pip_extent) == 20, "pip_extent offset mismatch"); +static_assert(offsetof(struct page_inv_page, pip_used) == 24, "pip_used offset mismatch"); +static_assert(offsetof(struct page_inv_page, pip_bits) == 28, "pip_bits offset mismatch"); + // SCN's Page @@ -502,6 +591,12 @@ struct scns_page ULONG scn_pages[1]; // SCN's vector }; +static_assert(sizeof(struct scns_page) == 24, "struct scns_page size mismatch"); +static_assert(offsetof(struct scns_page, scn_header) == 0, "scn_header offset mismatch"); +static_assert(offsetof(struct scns_page, scn_sequence) == 16, "scn_sequence offset mismatch"); +static_assert(offsetof(struct scns_page, scn_pages) == 20, "scn_pages offset mismatch"); + + // Important note ! // pagesPerPIP value must be multiply of pagesPerSCN value ! // @@ -544,6 +639,16 @@ struct pointer_page ULONG ppg_page[1]; // Data page vector }; +static_assert(sizeof(struct pointer_page) == 36, "struct pointer_page size mismatch"); +static_assert(offsetof(struct pointer_page, ppg_header) == 0, "ppg_header offset mismatch"); +static_assert(offsetof(struct pointer_page, ppg_sequence) == 16, "ppg_sequence offset mismatch"); +static_assert(offsetof(struct pointer_page, ppg_next) == 20, "ppg_next offset mismatch"); +static_assert(offsetof(struct pointer_page, ppg_count) == 24, "ppg_count offset mismatch"); +static_assert(offsetof(struct pointer_page, ppg_relation) == 26, "ppg_relation offset mismatch"); +static_assert(offsetof(struct pointer_page, ppg_min_space) == 28, "ppg_min_space offset mismatch"); +static_assert(offsetof(struct pointer_page, ppg_page) == 32, "ppg_page offset mismatch"); + + // pag_flags const UCHAR ppg_eof = 1; // Last pointer page in relation @@ -577,6 +682,11 @@ struct tx_inv_page UCHAR tip_transactions[1]; }; +static_assert(sizeof(struct tx_inv_page) == 24, "struct tx_inv_page size mismatch"); +static_assert(offsetof(struct tx_inv_page, tip_header) == 0, "tip_header offset mismatch"); +static_assert(offsetof(struct tx_inv_page, tip_next) == 16, "tip_next offset mismatch"); +static_assert(offsetof(struct tx_inv_page, tip_transactions) == 20, "tip_transactions offset mismatch"); + // Generator Page @@ -588,6 +698,12 @@ struct generator_page SINT64 gpg_values[1]; // Generator vector }; +static_assert(sizeof(struct generator_page) == 32, "struct generator_page size mismatch"); +static_assert(offsetof(struct generator_page, gpg_header) == 0, "gpg_header offset mismatch"); +static_assert(offsetof(struct generator_page, gpg_sequence) == 16, "gpg_sequence offset mismatch"); +static_assert(offsetof(struct generator_page, gpg_dummy1) == 20, "gpg_dummy1 offset mismatch"); +static_assert(offsetof(struct generator_page, gpg_values) == 24, "gpg_values offset mismatch"); + // Record header @@ -601,6 +717,14 @@ struct rhd UCHAR rhd_data[1]; // record data }; +static_assert(sizeof(struct rhd) == 16, "struct rhd size mismatch"); +static_assert(offsetof(struct rhd, rhd_transaction) == 0, "rhd_transaction offset mismatch"); +static_assert(offsetof(struct rhd, rhd_b_page) == 4, "rhd_b_page offset mismatch"); +static_assert(offsetof(struct rhd, rhd_b_line) == 8, "rhd_b_line offset mismatch"); +static_assert(offsetof(struct rhd, rhd_flags) == 10, "rhd_flags offset mismatch"); +static_assert(offsetof(struct rhd, rhd_format) == 12, "rhd_format offset mismatch"); +static_assert(offsetof(struct rhd, rhd_data) == 13, "rhd_data offset mismatch"); + #define RHD_SIZE static_cast(offsetof(Ods::rhd, rhd_data[0])) // Record header extended to hold long transaction id @@ -616,6 +740,15 @@ struct rhde UCHAR rhde_data[1]; // record data }; +static_assert(sizeof(struct rhde) == 20, "struct rhde size mismatch"); +static_assert(offsetof(struct rhde, rhde_transaction) == 0, "rhde_transaction offset mismatch"); +static_assert(offsetof(struct rhde, rhde_b_page) == 4, "rhde_b_page offset mismatch"); +static_assert(offsetof(struct rhde, rhde_b_line) == 8, "rhde_b_line offset mismatch"); +static_assert(offsetof(struct rhde, rhde_flags) == 10, "rhde_flags offset mismatch"); +static_assert(offsetof(struct rhde, rhde_format) == 12, "rhde_formats offset mismatch"); +static_assert(offsetof(struct rhde, rhde_tra_high) == 14, "rhde_tra_high offset mismatch"); +static_assert(offsetof(struct rhde, rhde_data) == 16, "rhde_data offset mismatch"); + #define RHDE_SIZE static_cast(offsetof(Ods::rhde, rhde_data[0])) // Record header for fragmented record @@ -633,6 +766,17 @@ struct rhdf UCHAR rhdf_data[1]; // record data }; +static_assert(sizeof(struct rhdf) == 24, "struct rhdf size mismatch"); +static_assert(offsetof(struct rhdf, rhdf_transaction) == 0, "rhdf_transaction offset mismatch"); +static_assert(offsetof(struct rhdf, rhdf_b_page) == 4, "rhdf_b_page offset mismatch"); +static_assert(offsetof(struct rhdf, rhdf_b_line) == 8, "rhdf_b_line offset mismatch"); +static_assert(offsetof(struct rhdf, rhdf_flags) == 10, "rhdf_flags offset mismatch"); +static_assert(offsetof(struct rhdf, rhdf_format) == 12, "rhdf_format offset mismatch"); +static_assert(offsetof(struct rhdf, rhdf_tra_high) == 14, "rhdf_tra_high offset mismatch"); +static_assert(offsetof(struct rhdf, rhdf_f_page) == 16, "rhdf_f_page offset mismatch"); +static_assert(offsetof(struct rhdf, rhdf_f_line) == 20, "rhdf_f_line offset mismatch"); +static_assert(offsetof(struct rhdf, rhdf_data) == 22, "rhdf_data offset mismatch"); + #define RHDF_SIZE static_cast(offsetof(Ods::rhdf, rhdf_data[0])) @@ -649,17 +793,29 @@ struct blh ULONG blh_length; // Total length of data USHORT blh_sub_type; // Blob sub-type UCHAR blh_charset; // Blob charset (since ODS 11.1) - UCHAR blh_unused; -#ifndef ODS_TESTING // Macro CHECK_BLOB_FIELD_ACCESS_FOR_SELECT is never defined, code under it was left for a case // we would like to have that check in a future. #ifdef CHECK_BLOB_FIELD_ACCESS_FOR_SELECT USHORT blh_fld_id; // Field ID #endif -#endif //ODS_TESTING + UCHAR blh_unused; ULONG blh_page[1]; // Page vector for blob pages }; +static_assert(sizeof(struct blh) == 32, "struct blh size mismatch"); +static_assert(offsetof(struct blh, blh_lead_page) == 0, "blh_lead_page offset mismatch"); +static_assert(offsetof(struct blh, blh_max_sequence) == 4, "blh_max_sequence offset mismatch"); +static_assert(offsetof(struct blh, blh_max_segment) == 8, "blh_max_segment offset mismatch"); +static_assert(offsetof(struct blh, blh_flags) == 10, "blh_flags offset mismatch"); +static_assert(offsetof(struct blh, blh_level) == 12, "blh_level offset mismatch"); +static_assert(offsetof(struct blh, blh_count) == 16, "blh_count offset mismatch"); +static_assert(offsetof(struct blh, blh_length) == 20, "blh_length offset mismatch"); +static_assert(offsetof(struct blh, blh_sub_type) == 24, "blh_sub_type offset mismatch"); +static_assert(offsetof(struct blh, blh_charset) == 26, "blh_charset offset mismatch"); +static_assert(offsetof(struct blh, blh_unused) == 27, "blh_unused offset mismatch"); +static_assert(offsetof(struct blh, blh_page) == 28, "blh_page offset mismatch"); + + #define BLH_SIZE static_cast(offsetof(Ods::blh, blh_page[0])) // rhd_flags, rhdf_flags and blh_flags @@ -691,6 +847,14 @@ struct Descriptor ULONG dsc_offset; }; +static_assert(sizeof(struct Descriptor) == 12, "struct Descriptor size mismatch"); +static_assert(offsetof(struct Descriptor, dsc_dtype) == 0, "dsc_dtype offset mismatch"); +static_assert(offsetof(struct Descriptor, dsc_scale) == 1, "dsc_scale offset mismatch"); +static_assert(offsetof(struct Descriptor, dsc_length) == 2, "dsc_length offset mismatch"); +static_assert(offsetof(struct Descriptor, dsc_sub_type) == 4, "dsc_sub_type offset mismatch"); +static_assert(offsetof(struct Descriptor, dsc_flags) == 6, "dsc_flags offset mismatch"); +static_assert(offsetof(struct Descriptor, dsc_offset) == 8, "dsc_offset offset mismatch"); + // Array description, "internal side" used by the engine. // And stored on the disk, in the relation summary blob. @@ -711,8 +875,24 @@ struct InternalArrayDesc SLONG iad_upper; // Upper bound }; iad_repeat iad_rpt[1]; + + static_assert(sizeof(struct iad_repeat) == 24, "struct iad_repeat size mismatch"); + static_assert(offsetof(struct iad_repeat, iad_desc) == 0, "iad_desc offset mismatch"); + static_assert(offsetof(struct iad_repeat, iad_length) == 12, "iad_length offset mismatch"); + static_assert(offsetof(struct iad_repeat, iad_lower) == 16, "iad_lower offset mismatch"); + static_assert(offsetof(struct iad_repeat, iad_upper) == 20, "iad_upper offset mismatch"); }; +static_assert(sizeof(struct InternalArrayDesc) == 40, "struct InternalArrayDesc size mismatch"); +static_assert(offsetof(struct InternalArrayDesc, iad_version) == 0, "iad_version offset mismatch"); +static_assert(offsetof(struct InternalArrayDesc, iad_dimensions) == 1, "iad_dimension offset mismatch"); +static_assert(offsetof(struct InternalArrayDesc, iad_struct_count) == 2, "iad_struct_count offset mismatch"); +static_assert(offsetof(struct InternalArrayDesc, iad_element_length) == 4, "iad_element_length offset mismatch"); +static_assert(offsetof(struct InternalArrayDesc, iad_length) == 6, "iad_length offset mismatch"); +static_assert(offsetof(struct InternalArrayDesc, iad_count) == 8, "iad_count offset mismatch"); +static_assert(offsetof(struct InternalArrayDesc, iad_total_length) == 12, "iad_total_length offset mismatch"); +static_assert(offsetof(struct InternalArrayDesc, iad_rpt) == 16, "iad_rpt offset mismatch"); + const UCHAR IAD_VERSION_1 = 1; /* @@ -727,13 +907,10 @@ inline int IAD_LEN(int count) #define IAD_LEN(count) (sizeof (Ods::InternalArrayDesc) + \ (count ? count - 1: count) * sizeof (Ods::InternalArrayDesc::iad_repeat)) -#ifndef ODS_TESTING Firebird::string pagtype(UCHAR type); -#endif //ODS_TESTING } //namespace Ods -#ifndef ODS_TESTING // alignment for raw page access const USHORT PAGE_ALIGNMENT = 1024; @@ -745,6 +922,5 @@ const int MAX_TABLE_VERSIONS = 255; // max number of view formats (aka versions), limited by "SSHORT RDB$FORMAT" const int MAX_VIEW_VERSIONS = MAX_SSHORT; -#endif //ODS_TESTING #endif // JRD_ODS_H diff --git a/src/misc/ods.awk b/src/misc/ods.awk deleted file mode 100644 index 53b6bb6451..0000000000 --- a/src/misc/ods.awk +++ /dev/null @@ -1,112 +0,0 @@ -# -# PROGRAM: ODS sizes and offsets validation -# MODULE: ods.awk -# DESCRIPTION: Generates c++ code printing reference sample for ODS strctures -# -# The contents of this file are subject to the Initial -# Developer's Public License Version 1.0 (the "License"); -# you may not use this file except in compliance with the -# License. You may obtain a copy of the License at -# http://www.firebirdsql.org/en/initial-developer-s-public-license-version-1-0/ -# -# Software distributed under the License is distributed AS IS, -# WITHOUT WARRANTY OF ANY KIND, either express or implied. -# See the License for the specific language governing rights -# and limitations under the License. -# -# The Original Code was created by Alexander Peshkoff -# for the Firebird Open Source RDBMS project. -# -# Copyright (c) 2018 Alexander Peshkoff -# and all contributors signed below. -# -# All Rights Reserved. -# Contributor(s): ______________________________________. -# - -BEGIN { - st = 0; - nm = ""; - nm2 = ""; - v2 = ""; - skip = 0; - - print "#include " - print "#ifdef _MSC_VER" - print "#include \"gen/autoconfig_msvc.h\"" - print "#else" - print "#include \"gen/autoconfig.h\"" - print "#endif" - print "" - print "#if SIZEOF_VOID_P == 8" - print "#define FMT \"l\"" - print "#else" - print "#define FMT \"\"" - print "#endif" - print "" - print "#include \"fb_types.h\"" - print "#define ODS_TESTING" - print "#include \"../jrd/ods.h\"" - print "" - print "using namespace Ods;" - print "" - print "int main() {" -} - -($1 == "struct") { - ++st; - if (st <= 2) - { - if (st == 1) - { - nm = $2; - nm2 = nm; - } - else - { - nm2 = nm "::" $2; - } - v2 = $2 "_"; - - print "\tprintf(\"\\n *** " $0 " %\" FMT \"u\\n\", sizeof(" nm2 "));"; - print "\t" nm2 " " v2 ";"; - } -} - - -($1 == "};") { - if (st > 1) - { - v2 = nm "_"; - } - st--; -} - -($1 == "}") { - if (st > 1) - { - st--; - v2 = nm "_"; - } -} - -($2 == "ODS_TESTING") { skip = 1; } - -(st > 0 && substr($1, 1, 1) != "#" && $1 != "struct" && skip == 0 && $1 != "//") { - m = $2; - i = index(m, "["); - if (i == 0) - i = index(m, ";"); - if (i > 0) - m = substr(m, 1, i - 1); - - if (length(m) > 0) - print "\tprintf(\"" m " %\" FMT \"d\\n\", (char*)&" v2 "." m " - (char*)&" v2 ");" -} - -($2 == "//ODS_TESTING") { skip = 0; } - -END { - print "}" -} - diff --git a/src/misc/ods.txt b/src/misc/ods.txt deleted file mode 100644 index 101bde4278..0000000000 --- a/src/misc/ods.txt +++ /dev/null @@ -1,185 +0,0 @@ - - *** struct pag 16 -pag_type 0 -pag_flags 1 -pag_reserved 2 -pag_generation 4 -pag_scn 8 -pag_pageno 12 - - *** struct blob_page 32 -blp_header 0 -blp_lead_page 16 -blp_sequence 20 -blp_length 24 -blp_pad 26 -blp_page 28 - - *** struct btree_page 40 -btr_header 0 -btr_sibling 16 -btr_left_sibling 20 -btr_prefix_total 24 -btr_relation 28 -btr_length 30 -btr_id 32 -btr_level 33 -btr_jump_interval 34 -btr_jump_size 36 -btr_jump_count 38 -btr_nodes 39 - - *** struct data_page 28 -dpg_header 0 -dpg_sequence 16 -dpg_relation 20 -dpg_count 22 - - *** struct dpg_repeat 4 -dpg_offset 0 -dpg_length 2 -dpg_rpt 24 - - *** struct index_root_page 32 -irt_header 0 -irt_relation 16 -irt_count 18 - - *** struct irt_repeat 12 -irt_root 0 -irt_transaction 4 -irt_desc 8 -irt_keys 10 -irt_flags 11 -irt_rpt 20 - - *** struct irtd 8 -irtd_field 0 -irtd_itype 2 -irtd_selectivity 4 - - *** struct header_page 132 -hdr_header 0 -hdr_page_size 16 -hdr_ods_version 18 -hdr_PAGES 20 -hdr_next_page 24 -hdr_oldest_transaction 28 -hdr_oldest_active 32 -hdr_next_transaction 36 -hdr_sequence 40 -hdr_flags 42 -hdr_creation_date 44 -hdr_attachment_id 52 -hdr_shadow_count 56 -hdr_cpu 60 -hdr_os 61 -hdr_cc 62 -hdr_compatibility_flags 63 -hdr_ods_minor 64 -hdr_end 66 -hdr_page_buffers 68 -hdr_oldest_snapshot 72 -hdr_backup_pages 76 -hdr_crypt_page 80 -hdr_crypt_plugin 84 -hdr_att_high 116 -hdr_tra_high 120 -hdr_data 128 - - *** struct page_inv_page 32 -pip_header 0 -pip_min 16 -pip_extent 20 -pip_used 24 -pip_bits 28 - - *** struct scns_page 24 -scn_header 0 -scn_sequence 16 -scn_pages 20 - - *** struct pointer_page 36 -ppg_header 0 -ppg_sequence 16 -ppg_next 20 -ppg_count 24 -ppg_relation 26 -ppg_min_space 28 -ppg_page 32 - - *** struct tx_inv_page 24 -tip_header 0 -tip_next 16 -tip_transactions 20 - - *** struct generator_page 32 -gpg_header 0 -gpg_sequence 16 -gpg_dummy1 20 -gpg_values 24 - - *** struct rhd 16 -rhd_transaction 0 -rhd_b_page 4 -rhd_b_line 8 -rhd_flags 10 -rhd_format 12 -rhd_data 13 - - *** struct rhde 20 -rhde_transaction 0 -rhde_b_page 4 -rhde_b_line 8 -rhde_flags 10 -rhde_format 12 -rhde_tra_high 14 -rhde_data 16 - - *** struct rhdf 24 -rhdf_transaction 0 -rhdf_b_page 4 -rhdf_b_line 8 -rhdf_flags 10 -rhdf_format 12 -rhdf_tra_high 14 -rhdf_f_page 16 -rhdf_f_line 20 -rhdf_data 22 - - *** struct blh 32 -blh_lead_page 0 -blh_max_sequence 4 -blh_max_segment 8 -blh_flags 10 -blh_level 12 -blh_count 16 -blh_length 20 -blh_sub_type 24 -blh_charset 26 -blh_unused 27 -blh_page 28 - - *** struct Descriptor 12 -dsc_dtype 0 -dsc_scale 1 -dsc_length 2 -dsc_sub_type 4 -dsc_flags 6 -dsc_offset 8 - - *** struct InternalArrayDesc 40 -iad_version 0 -iad_dimensions 1 -iad_struct_count 2 -iad_element_length 4 -iad_length 6 -iad_count 8 -iad_total_length 12 - - *** struct iad_repeat 24 -iad_desc 0 -iad_length 12 -iad_lower 16 -iad_upper 20 -iad_rpt 16 From 4fb0dfb6554fb74925d6f06d73e59dab9564e336 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 15 Jan 2020 00:04:26 +0000 Subject: [PATCH 217/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index f9937727b1..7c1443a4dd 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1723 + FORMAL BUILD NUMBER:1726 */ -#define PRODUCT_VER_STRING "4.0.0.1723" -#define FILE_VER_STRING "WI-T4.0.0.1723" -#define LICENSE_VER_STRING "WI-T4.0.0.1723" -#define FILE_VER_NUMBER 4, 0, 0, 1723 +#define PRODUCT_VER_STRING "4.0.0.1726" +#define FILE_VER_STRING "WI-T4.0.0.1726" +#define LICENSE_VER_STRING "WI-T4.0.0.1726" +#define FILE_VER_NUMBER 4, 0, 0, 1726 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1723" +#define FB_BUILD_NO "1726" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 73bb6e26c3..97e03e1981 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1723 +BuildNum=1726 NowAt=`pwd` cd `dirname $0` From 9164797892708585cc804b36ac7a4c2cbbfde064 Mon Sep 17 00:00:00 2001 From: Dimitry Sibiryakov Date: Wed, 15 Jan 2020 14:50:18 +0100 Subject: [PATCH 218/274] Refined Windows package creation (#251) --- .../win32/After_Installation.url | 2 +- .../win32/BuildExecutableInstall.bat | 408 +++++++----------- .../arch-specific/win32/FirebirdInstall.iss | 188 +++----- .../arch-specific/win32/Readme_DEV.txt | 2 + .../arch-specific/win32/i18n_readme.txt | 2 +- builds/install/misc/firebird.conf.in | 2 +- builds/win32/make_all.bat | 3 +- builds/win32/make_boot.bat | 2 +- builds/win32/make_examples.bat | 90 ++-- builds/win32/setenvvar.bat | 8 - configure.ac | 3 - src/CMakeLists.txt | 2 +- 12 files changed, 284 insertions(+), 428 deletions(-) diff --git a/builds/install/arch-specific/win32/After_Installation.url b/builds/install/arch-specific/win32/After_Installation.url index 430c312307..c5d29c42a6 100644 --- a/builds/install/arch-specific/win32/After_Installation.url +++ b/builds/install/arch-specific/win32/After_Installation.url @@ -1,2 +1,2 @@ [InternetShortcut] -URL=http://www.firebirdsql.org/afterinstall/$MAJOR$MINOR$RELEASE \ No newline at end of file +URL=http://www.firebirdsql.org/afterinstall/{#FB_MAJOR_VER}{#FB_MINOR_VER}{#FB_REV_NO} diff --git a/builds/install/arch-specific/win32/BuildExecutableInstall.bat b/builds/install/arch-specific/win32/BuildExecutableInstall.bat index db2b0426aa..2c0ba91ce9 100644 --- a/builds/install/arch-specific/win32/BuildExecutableInstall.bat +++ b/builds/install/arch-specific/win32/BuildExecutableInstall.bat @@ -22,6 +22,8 @@ ::============================================================================ :SET_PARAMS +::========= + @echo off :: reset ERRLEV to clear error from last run in same cmd shell set ERRLEV=0 @@ -35,21 +37,6 @@ set FBBUILD_ISX_PACK=0 if not defined FB2_SNAPSHOT (set FB2_SNAPSHOT=0) -:: Set our package number at 0 and increment every -:: time we rebuild in a single session -if not defined FBBUILD_PACKAGE_NUMBER ( -set FBBUILD_PACKAGE_NUMBER=0 -) else ( -set /A FBBUILD_PACKAGE_NUMBER+=1 -) -@echo Setting FBBUILD_PACKAGE_NUMBER to %FBBUILD_PACKAGE_NUMBER% - -::If a suffix is defined (usually for an RC) ensure it is prefixed correctly. -if defined FBBUILD_FILENAME_SUFFIX ( -if not "%FBBUILD_FILENAME_SUFFIX:~0,1%"=="_" ( -(set FBBUILD_FILENAME_SUFFIX=_%FBBUILD_FILENAME_SUFFIX%) -) -) :: See what we have on the command line @@ -61,39 +48,12 @@ for %%v in ( %* ) do ( ( if /I "%%v"=="ALL" ( (set FBBUILD_ZIP_PACK=1) & (set FBBUILD_ISX_PACK=1) ) ) ) -:: Now check whether we are debugging the InnoSetup script - -@if %FB2_ISS_DEBUG% equ 0 (@set ISS_BUILD_TYPE=iss_release) else (@set ISS_BUILD_TYPE=iss_debug) -@if %FB2_ISS_DEBUG% equ 0 (@set ISS_COMPRESS=compression) else (@set ISS_COMPRESS=nocompression) - -(@set ISS_EXAMPLES=examples) -@if %FB2_ISS_DEBUG% equ 1 ( - @if %FB2_EXAMPLES% equ 0 (@set ISS_EXAMPLES=noexamples) -) - ::Are we doing a snapshot build? If so we always do less work. if "%FB2_SNAPSHOT%"=="1" ( - (set FB_ISS_EXAMPLES=noexamples) (set FBBUILD_ISX_PACK=0) ) -:: Set up our final destination -set FBBUILD_INSTALL_IMAGES=%FB_ROOT_PATH%\builds\install_images -if not exist %FBBUILD_INSTALL_IMAGES% (mkdir %FBBUILD_INSTALL_IMAGES%) - - - -:: Determine Product Status -set FBBUILD_PROD_STATUS= -@type %FB_ROOT_PATH%\src\jrd\build_no.h | findstr /I UNSTABLE > nul && ( -set FBBUILD_PROD_STATUS=DEV) || type %FB_ROOT_PATH%\src\jrd\build_no.h | findstr /I ALPHA > nul && ( -set FBBUILD_PROD_STATUS=DEV) || type %FB_ROOT_PATH%\src\jrd\build_no.h | findstr /I BETA > nul && ( -set FBBUILD_PROD_STATUS=PROD) || type %FB_ROOT_PATH%\src\jrd\build_no.h | findstr /I "Release Candidate" > nul && ( -set FBBUILD_PROD_STATUS=PROD) || type %FB_ROOT_PATH%\src\jrd\build_no.h | findstr "RC" > nul && ( -set FBBUILD_PROD_STATUS=PROD) || type %FB_ROOT_PATH%\src\jrd\build_no.h | findstr /I "Final" > nul && ( -set FBBUILD_PROD_STATUS=PROD) - ::End of SET_PARAMS ::----------------- @goto :EOF @@ -110,6 +70,19 @@ set FBBUILD_PROD_STATUS=PROD) @echo o Checking for unix2dos... (cmd /c "unix2dos.exe --version 2>&1 | findstr version > nul" ) || ( call :ERROR Could not locate unix2dos && @goto :EOF ) +@for /f "usebackq tokens=*" %%c in (`where /f touch 2^>nul`) do set TOUCH_COMMAND=%%c +if defined TOUCH_COMMAND ( + @%TOUCH_COMMAND% --version nul 2>nul + if not errorlevel 1 ( + @echo o POSIX touch utility found at %TOUCH_COMMAND% + ) else ( @set TOUCH_COMMAND= ) +) + +@for /f "usebackq tokens=*" %%c in (`where /f md5sum 2^>nul`) do set MD5_COMMAND=%%c +if defined MD5_COMMAND ( + @echo o POSIX md5sum utility found at %MD5_COMMAND% +) + if %FBBUILD_ZIP_PACK% EQU 1 ( if not defined SEVENZIP ( call :ERROR SEVENZIP environment variable is not defined. @@ -117,33 +90,40 @@ if %FBBUILD_ZIP_PACK% EQU 1 ( ) else (@echo o Compression utility found.) ) +if %FBBUILD_ISX_PACK% NEQ 1 goto :SKIP_INNO -if %FBBUILD_ISX_PACK% EQU 1 ( - if NOT DEFINED INNO5_SETUP_PATH ( - call :ERROR INNO5_SETUP_PATH variable not defined - @goto :EOF - ) else ( - @echo o Inno Setup found at %INNO5_SETUP_PATH%. - ) +if defined INNO5_SETUP_PATH ( + set ISCC_COMMAND=%INNO5_SETUP_PATH%\iscc.exe ) +:: If the environment variable is not set let's search in PATH +if not defined ISCC_COMMAND ( + @for /f "usebackq tokens=*" %%c in (`where /f iscc 2^>nul`) do set ISCC_COMMAND=%%c +) +if not defined ISCC_COMMAND ( + @echo Required Inno Setup compiler not found + @exit /b 1 +) +@echo o Inno Setup found as %ISCC_COMMAND%. + +:SKIP_INNO if not defined WIX ( - call :ERROR WIX not defined. WiX is needed to build the MSI kits of the CRT runtimes. - @goto :EOF + @echo. + @echo The WIX environment var not defined. + @echo WiX is needed to build the MSI kits of the CRT runtimes. + @echo. ) else ( @echo o WiX found at "%WIX%". ) -if not DEFINED FB_EXTERNAL_DOCS ( +if not defined FB_EXTERNAL_DOCS ( @echo. @echo The FB_EXTERNAL_DOCS environment var is not defined @echo It should point to the directory containing the relevant release notes @echo in adobe pdf format. @echo. - @echo Subsequent script execution will be cancelled. - @echo. - cancel_script > nul 2>&1 - goto :EOF +) else ( + @echo o Package will include documentation from "%FB_EXTERNAL_DOCS%". ) @@ -152,94 +132,69 @@ if not DEFINED FB_EXTERNAL_DOCS ( @goto :EOF -:SED_MAGIC -:: Do some sed magic to make sure that the final product -:: includes the version string in the filename. -:: If the Firebird Unix tools for Win32 aren't on -:: the path this will fail! Use of the cygwin tools has not -:: been tested and may produce unexpected results. -::======================================================== -find "#define PRODUCT_VER_STRING" %FB_ROOT_PATH%\src\jrd\build_no.h > %temp%.\b$1.txt -sed -n -e s/\"//g -e s/"#define PRODUCT_VER_STRING "//w%temp%.\b$2.txt %temp%.\b$1.txt -for /f "tokens=*" %%a in ('type %temp%.\b$2.txt') do set FBBUILD_PRODUCT_VER_STRING=%%a +:SET_VERSION +::========== -find "#define FB_MAJOR_VER" %FB_ROOT_PATH%\src\jrd\build_no.h > %temp%.\b$1.txt -sed -n -e s/\"//g -e s/"#define FB_MAJOR_VER "//w%temp%.\b$2.txt %temp%.\b$1.txt -for /f "tokens=*" %%a in ('type %temp%.\b$2.txt') do set FB_MAJOR_VER=%%a +:: Cut off everything that is not #define to let Inno Setup use it +findstr /B /L "#define" "%FB_ROOT_PATH%\src\jrd\build_no.h" >"%FB_ROOT_PATH%\gen\jrd\build_no.h" +:: Read version parameters from build_no.h +for /F "tokens=2*" %%a in (%FB_ROOT_PATH%\gen\jrd\build_no.h) do ( +@echo Setting %%a to %%~b +SET %%a=%%~b +) -find "#define FB_MINOR_VER" %FB_ROOT_PATH%\src\jrd\build_no.h > %temp%.\b$1.txt -sed -n -e s/\"//g -e s/"#define FB_MINOR_VER "//w%temp%.\b$2.txt %temp%.\b$1.txt -for /f "tokens=*" %%a in ('type %temp%.\b$2.txt') do set FB_MINOR_VER=%%a +:: Set our package number at 0 and increment every +:: time we rebuild in a single session +if not defined FBBUILD_PACKAGE_NUMBER ( +set FBBUILD_PACKAGE_NUMBER=0 +) else ( +set /A FBBUILD_PACKAGE_NUMBER+=1 +) +@echo Setting FBBUILD_PACKAGE_NUMBER to %FBBUILD_PACKAGE_NUMBER% -find "#define FB_REV_NO" %FB_ROOT_PATH%\src\jrd\build_no.h > %temp%.\b$1.txt -sed -n -e s/\"//g -e s/"#define FB_REV_NO "//w%temp%.\b$2.txt %temp%.\b$1.txt -for /f "tokens=*" %%a in ('type %temp%.\b$2.txt') do set FB_REV_NO=%%a +:: If a suffix is defined (usually for an RC) ensure it is prefixed correctly. +if defined FBBUILD_FILENAME_SUFFIX ( +if not "%FBBUILD_FILENAME_SUFFIX:~0,1%"=="_" ( +(set FBBUILD_FILENAME_SUFFIX=_%FBBUILD_FILENAME_SUFFIX%) +) +) -find "#define FB_BUILD_NO" %FB_ROOT_PATH%\src\jrd\build_no.h > %temp%.\b$1.txt -sed -n -e s/\"//g -e s/"#define FB_BUILD_NO "//w%temp%.\b$2.txt %temp%.\b$1.txt -for /f "tokens=*" %%a in ('type %temp%.\b$2.txt') do set FB_BUILD_NO=%%a +:: Set up our final destination +set FBBUILD_INSTALL_IMAGES=%FB_ROOT_PATH%\builds\install_images +if not exist "%FBBUILD_INSTALL_IMAGES%" (mkdir "%FBBUILD_INSTALL_IMAGES%") -set FBBUILD_FILE_ID=%FBBUILD_PRODUCT_VER_STRING%-%FBBUILD_PACKAGE_NUMBER%_%FB_TARGET_PLATFORM% +:: Determine Product Status +if %FB_BUILD_TYPE%==V ( +set FBBUILD_PROD_STATUS=PROD +) else ( +set FBBUILD_PROD_STATUS=DEV +) -::@echo s/-2.0.0-/-%FBBUILD_PRODUCT_VER_STRING%-/ > %temp%.\b$3.txt -@echo s/define release/define %FBBUILD_BUILDTYPE%/ > %temp%.\b$3.txt -@echo s/define no_pdb/define %FBBUILD_SHIP_PDB%/ >> %temp%.\b$3.txt -::@echo s/define package_number=\"0\"/define package_number=\"%FBBUILD_PACKAGE_NUMBER%\"/ >> %temp%.\b$3.txt -@echo s/define iss_release/define %ISS_BUILD_TYPE%/ >> %temp%.\b$3.txt -@echo s/define examples/define %ISS_EXAMPLES%/ >> %temp%.\b$3.txt -@echo s/define compression/define %ISS_COMPRESS%/ >> %temp%.\b$3.txt -@echo s/FBBUILD_PRODUCT_VER_STRING/%FBBUILD_PRODUCT_VER_STRING%/ >> %temp%.\b$3.txt +set FBBUILD_FILE_ID=%PRODUCT_VER_STRING%-%FBBUILD_PACKAGE_NUMBER%_%FB_TARGET_PLATFORM% -sed -f %temp%.\b$3.txt FirebirdInstall.iss > FirebirdInstall_%FBBUILD_FILE_ID%.iss - -:: This is a better way of achieving what is done in make_all.bat, but we don't -:: test for sed in that script. -@sed /@UDF_COMMENT@/s/@UDF_COMMENT@/#/ < %FB_ROOT_PATH%\builds\install\misc\firebird.conf.in > %FB_OUTPUT_DIR%\firebird.conf - -set FBBUILD_FB40_CUR_VER=%FB_MAJOR_VER%.%FB_MINOR_VER%.%FB_REV_NO% -set FBBUILD_FB_CUR_VER=%FBBUILD_FB40_CUR_VER% -set FBBUILD_FB_LAST_VER=%FBBUILD_FB30_CUR_VER% - -:: Now set some version strings of our legacy releases. -:: This helps us copy the correct documentation, -:: as well as set up the correct shortcuts -set FBBUILD_FB15_CUR_VER=1.5.6 -set FBBUILD_FB20_CUR_VER=2.0.7 -set FBBUILD_FB21_CUR_VER=2.1.7 -set FBBUILD_FB25_CUR_VER=2.5.8 -set FBBUILD_FB30_CUR_VER=3.0.4 - -:: Now fix up the major.minor version strings in the readme files. -:: We place output in %FB_GEN_DIR%\readmes +@setlocal +@echo. @if not exist %FB_GEN_DIR%\readmes (@mkdir %FB_GEN_DIR%\readmes) -@for %%d in (ba cz de es fr hu it pl pt ru si ) do ( - @if not exist %FB_GEN_DIR%\readmes\%%d ( - @mkdir %FB_GEN_DIR%\readmes\%%d - ) -) - -@echo s/\$MAJOR/%FB_MAJOR_VER%/g > %temp%.\b$4.txt -@echo s/\$MINOR/%FB_MINOR_VER%/g >> %temp%.\b$4.txt -@echo s/\$RELEASE/%FB_REV_NO%/g >> %temp%.\b$4.txt -@echo %FBBUILD_PROD_STATUS% release. Copying Readme_%FBBUILD_PROD_STATUS%.txt Readme.txt -@copy Readme_%FBBUILD_PROD_STATUS%.txt Readme.txt -@for %%f in (Readme.txt installation_readme.txt After_Installation.url) do ( +set SED_COMMAND=sed -e s/\$MAJOR/%FB_MAJOR_VER%/g ^ + -e s/\$MINOR/%FB_MINOR_VER%/g ^ + -e s/\$RELEASE/%FB_REV_NO%/g +@echo Processing version strings in Readme_%FBBUILD_PROD_STATUS%.txt +@%SED_COMMAND% Readme_%FBBUILD_PROD_STATUS%.txt > %FB_GEN_DIR%\readmes\Readme.txt +@for %%f in (installation_readme.txt) do ( @echo Processing version strings in %%f - @sed -f %temp%.\b$4.txt %%f > %FB_GEN_DIR%\readmes\%%f + @%SED_COMMAND% %%f > %FB_GEN_DIR%\readmes\%%f ) @for %%d in (ba cz de es fr hu it pl pt ru si ) do ( - @pushd %%d - @for /F %%f in ( '@dir /B /A-D *.txt' ) do ( - @echo Processing version strings in %%d\%%f - @sed -f %temp%.\b$4.txt %%f > %FB_GEN_DIR%\readmes\%%d\%%f + @if not exist %FB_GEN_DIR%\readmes\%%d (@mkdir %FB_GEN_DIR%\readmes\%%d) + @for %%f in ( %%d\*.txt ) do ( + @echo Processing version strings in %%f + @%SED_COMMAND% %%f > %FB_GEN_DIR%\readmes\%%f ) - @popd ) -del %temp%.\b$?.txt +@endlocal - -::End of SED_MAGIC +::End of SET_VERSION ::---------------- @goto :EOF @@ -268,14 +223,14 @@ if %MSVC_VERSION% EQU 15 ( ) for %%f in ( msvcp%MSVC_RUNTIME_MAJOR_VERSION%%MSVC_RUNTIME_MINOR_VERSION_0%.dll vcruntime%MSVC_RUNTIME_MAJOR_VERSION%%MSVC_RUNTIME_MINOR_VERSION_0%.dll ) do ( echo Copying "%VCToolsRedistDir%\%VSCMD_ARG_TGT_ARCH%\Microsoft.VC%MSVC_RUNTIME_MAJOR_VERSION%%MSVC_RUNTIME_MINOR_VERSION_1%.CRT\%%f" - copy "%VCToolsRedistDir%\%VSCMD_ARG_TGT_ARCH%\Microsoft.VC%MSVC_RUNTIME_MAJOR_VERSION%%MSVC_RUNTIME_MINOR_VERSION_1%.CRT\%%f" %FB_OUTPUT_DIR%\ + copy "%VCToolsRedistDir%\%VSCMD_ARG_TGT_ARCH%\Microsoft.VC%MSVC_RUNTIME_MAJOR_VERSION%%MSVC_RUNTIME_MINOR_VERSION_1%.CRT\%%f" %FB_OUTPUT_DIR%\ >nul if %ERRORLEVEL% GEQ 1 ( call :ERROR Copying "%VCToolsRedistDir%\%VSCMD_ARG_TGT_ARCH%\Microsoft.VC%MSVC_RUNTIME_MAJOR_VERSION%%MSVC_RUNTIME_MINOR_VERSION_1%.CRT\%%f" failed with error %ERRORLEVEL% ) && (goto :EOF) ) ) -@implib.exe | findstr "Borland" > nul -@if errorlevel 0 ( +@where /Q implib.exe +@if not errorlevel 1 ( if "%VSCMD_ARG_TGT_ARCH%"=="x86" ( @echo Generating fbclient_bor.lib @implib %FB_OUTPUT_DIR%\lib\fbclient_bor.lib %FB_OUTPUT_DIR%\fbclient.dll > nul @@ -325,13 +280,8 @@ mkdir %FB_OUTPUT_DIR%\misc\upgrade\security 2>nul goto :EOF ) - @echo Copying other documentation... @copy %FB_GEN_DIR%\readmes\installation_readme.txt %FB_OUTPUT_DIR%\doc\installation_readme.txt > nul -:: WhatsNew doesn't exist at the moment (Alpha1) - perhaps it will turn up later in the release cycle. -:: In any case, if it is not an error if it doesn't exist -@ren %FB_OUTPUT_DIR%\doc\WhatsNew %FB_OUTPUT_DIR%\doc\WhatsNew.txt - :: FIX ME - we now have some .md files and ChangeLog is no longer a monster. :: Maybe we can just do nothing here. @@ -342,7 +292,6 @@ mkdir %FB_OUTPUT_DIR%\misc\upgrade\security 2>nul :: @copy %FB_ROOT_PATH%\ChangeLog %FB_OUTPUT_DIR%\doc\ChangeLog.txt > nul ::) - @mkdir %FB_OUTPUT_DIR%\doc\sql.extensions 2>nul @if %ERRORLEVEL% GEQ 2 ( (call :ERROR MKDIR for doc\sql.extensions dir failed) & (@goto :EOF)) @copy %FB_ROOT_PATH%\doc\sql.extensions\*.* %FB_OUTPUT_DIR%\doc\sql.extensions\ > nul @@ -353,7 +302,7 @@ mkdir %FB_OUTPUT_DIR%\misc\upgrade\security 2>nul :: if the docs are available then we can include them. if defined FB_EXTERNAL_DOCS ( @echo Copying pdf docs... -@for %%v in ( Firebird-%FB_MAJOR_VER%.%FB_MINOR_VER%-QuickStart.pdf Firebird_v%FBBUILD_FB_CUR_VER%.ReleaseNotes.pdf ) do ( +@for %%v in ( Firebird-%FB_MAJOR_VER%.%FB_MINOR_VER%-QuickStart.pdf Firebird_v%FB_MAJOR_VER%.%FB_MINOR_VER%.%FB_REV_NO%.ReleaseNotes.pdf ) do ( @echo ... %%v (@copy /Y %FB_EXTERNAL_DOCS%\%%v %FB_OUTPUT_DIR%\doc\%%v > nul) || (call :WARNING Copying %FB_EXTERNAL_DOCS%\%%v failed.) ) @@ -361,42 +310,46 @@ if defined FB_EXTERNAL_DOCS ( @echo. ) +@echo Cleaning irrelevant files... :: Clean out text notes that are either not relevant to Windows or :: are only of use to engine developers. @for %%v in ( README.makefiles README.user.embedded README.user.troubleshooting README.build.mingw.html README.build.msvc.html fb2-todo.txt cleaning-todo.txt install_win32.txt README.coding.style emacros-cross_ref.html firebird_conf.txt *.*~) do ( @del %FB_OUTPUT_DIR%\doc\%%v 2>nul ) +@echo Copy license... :: Add license for %%v in (IPLicense.txt IDPLicense.txt ) do ( @copy %FB_ROOT_PATH%\builds\install\misc\%%v %FB_OUTPUT_DIR%\%%v > nul ) :: And readme -@copy %FB_GEN_DIR%\readmes\readme.txt %FB_OUTPUT_DIR%\ > nul +@copy %FB_GEN_DIR%\readmes\Readme.txt %FB_OUTPUT_DIR%\ > nul :: Walk through all docs and transform any that are not .txt, .pdf or .html to .txt @echo Setting .txt filetype to ascii docs. -for /R %FB_OUTPUT_DIR%\doc %%v in (.) do ( - pushd %%v - for /F %%W in ( 'dir /B /A-D' ) do ( - if /I "%%~xW" NEQ ".txt" ( - if /I "%%~xW" NEQ ".pdf" ( - if /I "%%~xW" NEQ ".htm" ( - if /I "%%~xW" NEQ ".html" ( - ren %%W %%W.txt +for /R %FB_OUTPUT_DIR%\doc %%v in ( * ) do ( + if /I not "%%~xv" == ".md" ( + if /I not "%%~xv" == ".txt" ( + if /I not "%%~xv" == ".pdf" ( + if /I not "%%~xv" == ".htm" ( + if /I not "%%~xv" == ".html" ( + ren %%v %%~nxv.txt ) ) ) ) ) - popd ) -:: Throw away any errorlevel left hanging around -@set | findstr win > nul +if %FB2_SNAPSHOT% EQU 1 ( + @copy %FB_ROOT_PATH%\builds\install\arch-specific\win32\readme_snapshot.txt %FB_OUTPUT_DIR%\readme_snapshot.txt > nul +) @echo Completed copying docs. + +:: Examples were already copied by make_examples + ::End of COPY_XTRA ::---------------- @goto :EOF @@ -426,6 +379,7 @@ if %MSVC_VERSION% EQU 15 ( :INCLUDE_DIR +::========== :: Prepare other files needed for deployment to /include dir setlocal :: grab some missing bits'n'pieces from different parts of the source tree @@ -437,8 +391,8 @@ setlocal @echo Copying other include files required for development... set OUTPATH=%FB_OUTPUT_DIR%\include @copy %FB_ROOT_PATH%\src\yvalve\perf.h %OUTPATH%\ > nul -@copy %FB_ROOT_PATH%\src\include\gen\firebird.pas %OUTPATH%\firebird\ > nul || (@call :ERROR Failure executing copy %FB_ROOT_PATH%\src\include\gen\firebird.pas %OUTPATH%\firebird\ && @goto :EOF ) -@xcopy /e /i /y %FB_ROOT_PATH%\src\include\firebird\impl %OUTPATH%\firebird\ > nul || (@call :ERROR Failure executing @xcopy /e /i /y %FB_ROOT_PATH%\src\include\firebird\* %OUTPATH%\firebird\ && @goto :EOF ) +@copy %FB_ROOT_PATH%\src\include\gen\firebird.pas %OUTPATH%\firebird\ > nul || (@call :ERROR Failure executing copy %FB_ROOT_PATH%\src\include\gen\firebird.pas %OUTPATH%\firebird\ ) +@xcopy /e /i /y %FB_ROOT_PATH%\src\include\firebird\impl %OUTPATH%\firebird\ > nul || (@call :ERROR Failure executing @xcopy /e /i /y %FB_ROOT_PATH%\src\include\firebird\* %OUTPATH%\firebird\ ) @if %ERRLEV% GEQ 1 goto :END endlocal @@ -494,85 +448,40 @@ copy %FB_ROOT_PATH%\builds\install\misc\databases.conf.in %FB_OUTPUT_DIR%\databa :: Get a list of all files in the tree make sure :: that and they all have windows EOL ::=============================================== -for /F %%W in ( 'dir %FB_OUTPUT_DIR% /b /a-d /s' ) do ( - for %%X in ( txt conf sql c cpp hpp h bat pas e def rc md ) do ( - if /I "%%~xW" EQU ".%%X" ( unix2dos --u2d --safe %%W 2>nul >nul ) - ) +for /R %FB_OUTPUT_DIR% %%W in ( *.txt *.conf *.sql *.c *.cpp *.hpp *.h *.bat *.pas *.e *.def *.rc *.md *.html ) do ( + unix2dos -q --safe %%W || exit /b 1 ) + ::End of SET_CRLF ::------------- @goto :EOF -:GEN_ZIP -::====== -if %FBBUILD_ZIP_PACK% EQU 0 goto :EOF -@echo - Generate the directory tree to be zipped -set FBBUILD_ZIP_PACK_ROOT=%FB_ROOT_PATH%\builds\zip_pack_%FB_TARGET_PLATFORM% -if not exist %FBBUILD_ZIP_PACK_ROOT% @mkdir %FBBUILD_ZIP_PACK_ROOT% 2>nul -@del /s /q %FBBUILD_ZIP_PACK_ROOT%\ > nul -::@copy /Y %FB_OUTPUT_DIR% %FBBUILD_ZIP_PACK_ROOT% > nul -::for %%v in (doc doc\sql.extensions help include intl lib udf misc misc\upgrade\security plugins system32 ) do ( -:: @mkdir %FBBUILD_ZIP_PACK_ROOT%\%%v 2>nul -:: @dir /b /a-d /s %FB_OUTPUT_DIR%\%%v\*.* >nul 2>nul -:: if not ERRORLEVEL 1 @copy /Y %FB_OUTPUT_DIR%\%%v\*.* %FBBUILD_ZIP_PACK_ROOT%\%%v\ > nul -::) -@xcopy /Y /E /S %FB_OUTPUT_DIR% %FBBUILD_ZIP_PACK_ROOT% > nul - -@echo - Add examples to zip tree -@xcopy /Y /E /S %FB_OUTPUT_DIR%\examples\*.* %FBBUILD_ZIP_PACK_ROOT%\examples > nul -::@if %FB2_EXAMPLES% equ 1 for %%v in (examples examples\api examples\build_win32 examples\dbcrypt examples\empbuild examples\include examples\interfaces examples\package examples\stat examples\udf examples\udr ) do ( -:: @mkdir %FBBUILD_ZIP_PACK_ROOT%\%%v 2>nul -:: dir %FB_OUTPUT_DIR%\%%v\*.* > nul 2>nul -:: if not ERRORLEVEL 1 @copy /Y %FB_OUTPUT_DIR%\%%v\*.* %FBBUILD_ZIP_PACK_ROOT%\%%v\ > nul -::) - - -@echo - Now remove stuff from zip tree that is not needed... -setlocal -set FB_RM_FILE_LIST=doc\installation_readme.txt system32\vccrt%MSVC_VERSION%_%FB_TARGET_PLATFORM%.wixpdb icudt52l_empty.dat -for %%v in ( %FB_RM_FILE_LIST% ) do ( - @del %FBBUILD_ZIP_PACK_ROOT%\%%v > nul 2>&1 -) -endlocal - -if %FB2_SNAPSHOT% EQU 1 ( - @copy %FB_ROOT_PATH%\builds\install\arch-specific\win32\readme_snapshot.txt %FBBUILD_ZIP_PACK_ROOT%\readme_snapshot.txt > nul -) - -if not "%FBBUILD_SHIP_PDB%"=="ship_pdb" ( - @del /q %FBBUILD_ZIP_PACK_ROOT%\*.pdb > nul 2>&1 -) - -rmdir /s /q %FBBUILD_ZIP_PACK_ROOT%\examples\build_unix - -:: Don't grab old install notes for zip pack - document needs a complete re-write. -::@copy %FB_ROOT_PATH%\doc\install_win32.txt %FBBUILD_ZIP_PACK_ROOT%\doc\README_installation.txt > nul - -::End of GEN_ZIP -::-------------- -goto :EOF - - :ZIP_PACK ::======= -if %FBBUILD_ZIP_PACK% EQU 0 goto :EOF -if "%FBBUILD_SHIP_PDB%" == "ship_pdb" ( - if exist %FBBUILD_INSTALL_IMAGES%\Firebird-%FBBUILD_FILE_ID%_pdb%FBBUILD_FILENAME_SUFFIX%.zip ( - @del %FBBUILD_INSTALL_IMAGES%\Firebird-%FBBUILD_FILE_ID%_pdb%FBBUILD_FILENAME_SUFFIX%.zip - ) - set FBBUILD_ZIPFILE=%FBBUILD_INSTALL_IMAGES%\Firebird-%FBBUILD_FILE_ID%_pdb%FBBUILD_FILENAME_SUFFIX%.zip +:: Forcefully disable delayed expansion because of exclamation marks in 7z switches +setlocal DisableDelayedExpansion +set SKIP_FILES=-x!installation_readme.txt + +if "%FBBUILD_SHIP_PDB%" == "ship_pdb" ( + set FBBUILD_ZIPFILE=%FBBUILD_INSTALL_IMAGES%\Firebird-%FBBUILD_FILE_ID%_pdb%FBBUILD_FILENAME_SUFFIX%.zip ) else ( - if exist %FBBUILD_INSTALL_IMAGES%\Firebird-%FBBUILD_FILE_ID%%FBBUILD_FILENAME_SUFFIX%.zip ( - @del %FBBUILD_INSTALL_IMAGES%\Firebird-%FBBUILD_FILE_ID%%FBBUILD_FILENAME_SUFFIX%.zip - ) set FBBUILD_ZIPFILE=%FBBUILD_INSTALL_IMAGES%\Firebird-%FBBUILD_FILE_ID%%FBBUILD_FILENAME_SUFFIX%.zip + set SKIP_FILES=%SKIP_FILES% -x!*.pdb ) -@%SEVENZIP%\7z.exe a -r -tzip -mx9 %FBBUILD_ZIPFILE% %FBBUILD_ZIP_PACK_ROOT%\*.* -@echo End of ZIP_PACK -@echo. +if "%FB2_EXAMPLES%" == "0" set SKIP_FILES=%SKIP_FILES% -xr-!examples + +if exist %FBBUILD_ZIPFILE% ( + @del %FBBUILD_ZIPFILE% +) + +%SEVENZIP%\7z.exe a -r -tzip -mx9 %SKIP_FILES% %FBBUILD_ZIPFILE% %FB_OUTPUT_DIR%\*.* + +endlocal + +::End of ZIP_PACK ::---------------- @goto :EOF @@ -583,12 +492,23 @@ if "%FBBUILD_SHIP_PDB%" == "ship_pdb" ( ::While building and testing this feature might be annoying, so we don't do it. ::========================================================== setlocal -set TIMESTRING=0%PRODUCT_VER_STRING:~0,1%:0%PRODUCT_VER_STRING:~2,1%:0%PRODUCT_VER_STRING:~4,1% -@if /I "%BUILDTYPE%"=="release" ( - (@echo Touching release build files with %TIMESTRING% timestamp) & (touch -s -D -t%TIMESTRING% %FB_OUTPUT_DIR%\*.*) - (if %FBBUILD_ZIP_PACK% EQU 1 (@echo Touching release build files with %TIMESTRING% timestamp) & (touch -s -D -t%TIMESTRING% %FB_ROOT_PATH%\zip_pack\*.*) ) + +if /I not "%FBBUILD_BUILDTYPE%"=="release" goto :EOF +if not defined TOUCH_COMMAND echo POSIX touch utility not found && exit /b 1 + +set TIMESTRING=0%FB_MAJOR_VER%:0%FB_MINOR_VER%:0%FB_REV_NO% + +:: Perhaps here we should touch directories as well +:: Here and there XXX_COMMAND is "call"-ed in case if it is a batch file + +@echo Touching release build files with %TIMESTRING% timestamp + +@for /R %FB_OUTPUT_DIR% %%F in ( * ) do ( + call %TOUCH_COMMAND% -c -d %TIMESTRING% %%F || exit /b 1 ) + endlocal + ::End of TOUCH_ALL ::---------------- @goto :EOF @@ -602,14 +522,11 @@ endlocal :: eg set INNO5_SETUP_PATH="C:\Program Files\Inno Setup 5" :: ::================================================= -if %FBBUILD_ISX_PACK% NEQ 1 goto :EOF -@echo Now let's compile the InnoSetup scripts @echo. -%INNO5_SETUP_PATH%\iscc %FB_ROOT_PATH%\builds\install\arch-specific\win32\FirebirdInstall_%FBBUILD_FILE_ID%.iss +call %ISCC_COMMAND% %FB_ROOT_PATH%\builds\install\arch-specific\win32\FirebirdInstall.iss @echo. -@echo End of ISX_PACK -@echo. +::End of ISX_PACK ::--------------- @goto :EOF @@ -618,15 +535,25 @@ if %FBBUILD_ISX_PACK% NEQ 1 goto :EOF ::========= :: Generate the md5sum checksum file ::================================== -if NOT DEFINED GNU_TOOLCHAIN ( - call :WARNING GNU_TOOLCHAIN variable not defined. Cannot generate md5 sums. +if not defined MD5_COMMAND ( + call :WARNING md5sum utility not found. Cannot generate md5 sums. @goto :EOF ) -@echo Generating md5sums for Firebird-%FBBUILD_PRODUCT_VER_STRING%-%FBBUILD_PACKAGE_NUMBER% +@echo Generating md5sums for Firebird-%PRODUCT_VER_STRING%-%FBBUILD_PACKAGE_NUMBER% -%GNU_TOOLCHAIN%\md5sum.exe %FBBUILD_INSTALL_IMAGES%\Firebird-%FBBUILD_PRODUCT_VER_STRING%?%FBBUILD_PACKAGE_NUMBER%*.* > %FBBUILD_INSTALL_IMAGES%\Firebird-%FBBUILD_PRODUCT_VER_STRING%-%FBBUILD_PACKAGE_NUMBER%.md5sum +:: write sums into temporary file to avoid including it into the process +pushd %FBBUILD_INSTALL_IMAGES% +call %MD5_COMMAND% Firebird-%PRODUCT_VER_STRING%?%FBBUILD_PACKAGE_NUMBER%*.* >md5sum.tmp -::--------------- +:: then rename it to the proper name +if not errorlevel 1 ( + del Firebird-%PRODUCT_VER_STRING%-%FBBUILD_PACKAGE_NUMBER%.md5sum >nul 2>nul + ren md5sum.tmp Firebird-%PRODUCT_VER_STRING%-%FBBUILD_PACKAGE_NUMBER%.md5sum +) +popd + +::End of DO_MD5SUMS +::----------------- @goto :EOF @@ -654,7 +581,7 @@ if NOT DEFINED GNU_TOOLCHAIN ( @echo. @echo HELP This help screen. @echo. -@echo In addition, the following environment variables are checked: +@echo In addition, the following environment variables are checked by ISS script: @echo. @echo FB2_ISS_DEBUG=1 - Prepare an InnoSetup script that is @echo easier to debug @@ -729,7 +656,6 @@ popd @if errorlevel 1 (goto :END) @if not defined FB2_ISS_DEBUG (set FB2_ISS_DEBUG=0) -@if not defined FB2_EXAMPLES (set FB2_EXAMPLES=1) @echo. @echo Reading command-line parameters... @@ -742,7 +668,7 @@ popd @echo. @echo Setting version number... -@(@call :SED_MAGIC ) || (@echo Error calling SED_MAGIC && @goto :END) +@(@call :SET_VERSION ) || (@echo Error calling SET_VERSION && @goto :END) @echo. @echo Copying additional files needed for installation, documentation etc. @@ -776,13 +702,6 @@ if defined WIX ( @(@call :SET_CRLF ) || (@echo Error calling SET_CRLF && @goto :EOF) @echo. - -if %FBBUILD_ZIP_PACK% EQU 1 ( -@echo Generating image of zipped install -@(@call :GEN_ZIP ) || (@echo Error calling GEN_ZIP && @goto :END) -@echo. -) - ::@echo Creating .local files for libraries ::@(@call :TOUCH_LOCAL ) || (@echo Error calling TOUCH_LOCAL & @goto :END) ::@echo. @@ -797,6 +716,7 @@ if %FBBUILD_ZIP_PACK% EQU 1 ( ) if %FBBUILD_ISX_PACK% EQU 1 ( +@echo Now let's compile the InnoSetup scripts @(@call :ISX_PACK ) || (@echo Error calling ISX_PACK && @goto :END) @echo. ) @@ -815,7 +735,7 @@ if %FBBUILD_ISX_PACK% EQU 1 ( :END -popd + exit /b diff --git a/builds/install/arch-specific/win32/FirebirdInstall.iss b/builds/install/arch-specific/win32/FirebirdInstall.iss index 57840935ee..49c7557776 100644 --- a/builds/install/arch-specific/win32/FirebirdInstall.iss +++ b/builds/install/arch-specific/win32/FirebirdInstall.iss @@ -49,30 +49,46 @@ #define FirebirdURL MyAppURL #define UninstallBinary "{app}\firebird.exe" +#define Root GetEnv("FB_ROOT_PATH") +#if Root == "" +;We are not run from batch file, let's set some sane defaults +#define Root = "..\..\..\.." +;Assume iss debug as well +#define iss_debug +#else + +#endif + +#if GetEnv("FB2_ISS_DEBUG") == "1" +#define iss_debug +#endif + +#if GetEnv("FBBUILD_SHIP_PDB") == "ship_pdb" +#define ship_pdb +#endif + +;Get version information from build_no.h +#include Root + "\gen\jrd\build_no.h" + ;Hard code some defaults to aid debugging and running script standalone. ;In practice, these values are set in the environment and we use the env vars. -#define MajorVer "4" -#define MinorVer "0" -#define PointRelease "0" -#define BuildNumber "0" +#define PackageNumber GetEnv("FBBUILD_PACKAGE_NUMBER") +#if PackageNumber == "" #define PackageNumber "0" -#define FilenameSuffix "" - +#endif +#define FilenameSuffix GetEnv("FBBUILD_FILENAME_SUFFIX") +#if FilenameSuffix != "" && pos('_',FilenameSuffix) == 0 +#define FilenameSuffix "_" + FilenameSuffix +#endif ;-------Start of Innosetup script debug flags section -; if iss_release is undefined then iss_debug is set +; if iss_debug is undefined then iss_release is set ; Setting iss_release implies that the defines for files, ; examples and compression are set. If debug is set then this ; section controls the settings of files, examples ; and compression. -; A dynamically generated sed script sets the appropriate define -; See BuildExecutableInstall.bat for more details. - - -;#define iss_debug - #ifndef iss_debug #define iss_release #endif @@ -80,10 +96,11 @@ ;;;;;;;;;;;;;;;;;;;;;;;;; #ifdef iss_release #define files +#if GetEnv("FB2_EXAMPLES") != "0" #define examples +#endif #define compression -#else -#define iss_debug +#define i18n #endif ;-------------------- @@ -107,72 +124,8 @@ ;-------end of Innosetup script debug flags section - ;-------Start of Innosetup script - -;---- These three defines need a bit of tidying up in the near future, -; but for now they must stay, as the BuildExecutableInstall.bat -; uses them. -#define release -#define no_pdb -#define i18n - - -;------If necessary we can turn off i18n by uncommenting this undefine -;------In general this is a good idea for alpha and beta releases. -#undef i18n - -;----- If we are debugging the script (and not executed from command prompt) -;----- there is no guarantee that the environment variable exists. However an -;----- expression such as #define FB_MAJOR_VER GetEnv("FB_MAJOR_VER") will -;----- 'define' the macro anyway so we need to test for a valid env var before -;----- we define our macro. -#if Len(GetEnv("FB_MAJOR_VER")) > 0 -#define FB_MAJOR_VER GetEnv("FB_MAJOR_VER") -#endif -#ifdef FB_MAJOR_VER -#define MajorVer FB_MAJOR_VER -#endif - -#if Len(GetEnv("FB_MINOR_VER")) > 0 -#define FB_MINOR_VER GetEnv("FB_MINOR_VER") -#endif -#ifdef FB_MINOR_VER -#define MinorVer FB_MINOR_VER -#endif - -#if Len(GetEnv("FB_REV_NO")) > 0 -#define FB_REV_NO GetEnv("FB_REV_NO") -#endif -#ifdef FB_REV_NO -#define PointRelease FB_REV_NO -#endif - -#if Len(GetEnv("FB_BUILD_NO")) > 0 -#define FB_BUILD_NO GetEnv("FB_BUILD_NO") -#endif -#ifdef FB_BUILD_NO -#define BuildNumber FB_BUILD_NO -#endif - -#if Len(GetEnv("FBBUILD_PACKAGE_NUMBER")) > 0 -#define FBBUILD_PACKAGE_NUMBER GetEnv("FBBUILD_PACKAGE_NUMBER") -#endif -#ifdef FBBUILD_PACKAGE_NUMBER -#define PackageNumber FBBUILD_PACKAGE_NUMBER -#endif - -#if Len(GetEnv("FBBUILD_FILENAME_SUFFIX")) > 0 -#define FBBUILD_FILENAME_SUFFIX GetEnv("FBBUILD_FILENAME_SUFFIX") -#endif -#ifdef FBBUILD_FILENAME_SUFFIX -#define FilenameSuffix FBBUILD_FILENAME_SUFFIX -#if pos('_',FilenameSuffix) == 0 -#define FilenameSuffix "_" + FilenameSuffix -#endif -#endif - #if Len(GetEnv("MSVC_VERSION")) > 0 #define msvc_version GetEnv("MSVC_VERSION") #else @@ -212,10 +165,10 @@ #define msvcr_filename = "vcruntime" #endif -#if BuildNumber == "0" -#define MyAppVerString MajorVer + "." + MinorVer + "." + PointRelease +#if FB_BUILD_NO == "0" +#define MyAppVerString FB_MAJOR_VER + "." + FB_MINOR_VER + "." + FB_REV_NO #else -#define MyAppVerString MajorVer + "." + MinorVer + "." + PointRelease + "." + BuildNumber +#define MyAppVerString FB_MAJOR_VER + "." + FB_MINOR_VER + "." + FB_REV_NO + "." + FB_BUILD_NO #endif #define MyAppVerName MyAppName + " " + MyAppVerString @@ -224,11 +177,16 @@ #define PlatformTarget GetEnv("FB_TARGET_PLATFORM") #endif #if PlatformTarget == "" +;Assume native platform +#if IsWin64 +#define PlatformTarget "x64" +#else #define PlatformTarget "win32" #endif +#endif +#if FB_BUILD_TYPE == "T" ;If we are still under development we can ignore some missing files. -#if GetEnv("FBBUILD_PROD_STATUS") == "DEV" #define SkipFileIfDevStatus " skipifsourcedoesntexist " #else #define SkipFileIfDevStatus " " @@ -240,21 +198,12 @@ #define WOW64Dir="output_win32" #endif -;BaseVer should be used for all MajorVer.MinorVer installs. -;This allows us to upgrade silently from MajorVer.MinorVer.m to MajorVer.MinorVer.n -#define BaseVer MajorVer + "_" + MinorVer -#define AppVer MajorVer + "_" + MinorVer -#define GroupnameVer MajorVer + "." + MinorVer - -;These variables are set in BuildExecutableInstall -#define FB15_cur_ver GetEnv("FBBUILD_FB15_CUR_VER") -#define FB20_cur_ver GetEnv("FBBUILD_FB20_CUR_VER") -#define FB21_cur_ver GetEnv("FBBUILD_FB21_CUR_VER") -#define FB25_cur_ver GetEnv("FBBUILD_FB25_CUR_VER") -#define FB30_cur_ver GetEnv("FBBUILD_FB30_CUR_VER") -#define FB40_cur_ver GetEnv("FBBUILD_FB40_CUR_VER") -#define FB_cur_ver FB40_cur_ver -#define FB_last_ver FB30_cur_ver +;BaseVer should be used for all FB_MAJOR_VER.FB_MINOR_VER installs. +;This allows us to upgrade silently from FB_MAJOR_VER.FB_MINOR_VER.m to FB_MAJOR_VER.FB_MINOR_VER.n +#define BaseVer FB_MAJOR_VER + "_" + FB_MINOR_VER +#define AppVer FB_MAJOR_VER + "_" + FB_MINOR_VER +#define GroupnameVer FB_MAJOR_VER + "." + FB_MINOR_VER +#define FB_cur_ver FB_MAJOR_VER + "." + FB_MINOR_VER + "." + FB_REV_NO ; We can save space by shipping a pdb package that just includes ; the pdb files. It would then upgrade an existing installation, @@ -276,7 +225,7 @@ #else #define pdb_str="" #endif -#ifdef debug +#if GetEnv("FBBUILD_BUILDTYPE") == "debug" #define debug_str="_debug" #else #define debug_str="" @@ -300,7 +249,7 @@ AppUpdatesURL={#MyAppURL} AppVersion={#MyAppVerString} VersionInfoVersion={#MyAppVerString} -SourceDir=..\..\..\..\ +SourceDir={#Root} OutputBaseFilename={#MyAppName}-{#MyAppVerString}_{#PackageNumber}_{#PlatformTarget}{#debug_str}{#pdb_str}{#FilenameSuffix} ;OutputManifestFile={#MyAppName}-{#MyAppVerString}_{#PackageNumber}_{#PlatformTarget}{#debug_str}{#pdb_str}{#FilenameSuffix}_Setup-Manifest.txt OutputDir=builds\install_images @@ -342,7 +291,7 @@ SetupLogging=yes #endif [Languages] -Name: en; MessagesFile: compiler:Default.isl; InfoBeforeFile: {#GenDir}\installation_readme.txt; InfoAfterFile: {#GenDir}\readme.txt; +Name: en; MessagesFile: compiler:Default.isl; InfoBeforeFile: {#GenDir}\installation_readme.txt; InfoAfterFile: {#GenDir}\Readme.txt; #ifdef i18n Name: ba; MessagesFile: compiler:Languages\Bosnian.isl; InfoBeforeFile: {#GenDir}\ba\Instalacija_ProcitajMe.txt; InfoAfterFile: {#GenDir}\ba\ProcitajMe.txt; Name: cz; MessagesFile: compiler:Languages\Czech.isl; InfoBeforeFile: {#GenDir}\cz\instalace_ctime.txt; InfoAfterFile: {#GenDir}\cz\ctime.txt; @@ -389,21 +338,6 @@ Name: ru; MessagesFile: compiler:Languages\Russian.isl; InfoBeforeFile: {#GenDir ;#include "si\custom_messages_si.inc" #endif -#ifdef iss_debug -; *** Note - this comment section needs revision or deletion. -; It is only applicable to the ansi installer, which is no longer -; supported for Firebird 3 -; By default, the languages available at runtime depend on the user's -; code page. A user with the Western European code page set will not -; even see that we support installation with the czech language -; for example. -; It can be useful when debugging to force the display of all available -; languages by setting LanguageCodePage to 0. Of course, if the langauge -; is not supported by the user's current code page it will be unusable. -[LangOptions] -LanguageCodePage=0 -#endif - [Types] Name: ServerInstall; Description: {cm:ServerInstall} Name: DeveloperInstall; Description: {cm:DeveloperInstall} @@ -478,7 +412,7 @@ Name: {group}\Firebird Server; Filename: {app}\firebird.exe; Parameters: {code:S Name: {group}\Firebird Guardian; Filename: {app}\fbguard.exe; Parameters: {code:StartAppParams}; Flags: runminimized; MinVersion: 4.0,4.0; Check: InstallGuardianIcon; IconIndex: 1; Components: ServerComponent; Comment: Run Firebird Server (with guardian); Name: {group}\Firebird ISQL Tool; Filename: {app}\isql.exe; Parameters: -z; WorkingDir: {app}; MinVersion: 4.0,4.0; Comment: {cm:RunISQL} Name: {group}\Firebird {#FB_cur_ver} Release Notes; Filename: {app}\doc\Firebird_v{#FB_cur_ver}.ReleaseNotes.pdf; MinVersion: 4.0,4.0; Comment: {#MyAppName} {cm:ReleaseNotes} -;Name: {group}\Firebird {#GroupnameVer} Quick Start Guide; Filename: {app}\doc\Firebird-{#MajorVer}-QuickStart.pdf; MinVersion: 4.0,4.0; Comment: {#MyAppName} {#FB_cur_ver} +;Name: {group}\Firebird {#GroupnameVer} Quick Start Guide; Filename: {app}\doc\Firebird-{#FB_MAJOR_VER}-QuickStart.pdf; MinVersion: 4.0,4.0; Comment: {#MyAppName} {#FB_cur_ver} Name: "{group}\After Installation"; Filename: "{app}\doc\After_Installation.url"; Comment: "New User? Here's a quick guide to what you should do next." Name: "{group}\Firebird Web-site"; Filename: "{app}\doc\firebirdsql.org.url" ;Always install the original english version @@ -493,7 +427,7 @@ Name: {group}\{cm:Uninstall,{#FB_cur_ver}}; Filename: {uninstallexe}; Comment: U #ifdef files Source: {#LicensesDir}\IPLicense.txt; DestDir: {app}; Components: ClientComponent; Flags: sharedfile ignoreversion; Source: {#LicensesDir}\IDPLicense.txt; DestDir: {app}; Components: ClientComponent; Flags: sharedfile ignoreversion -Source: {#ScriptsDir}\After_Installation.url; DestDir: {app}\doc; Components: ServerComponent DevAdminComponent; Flags: sharedfile ignoreversion +Source: {#file "After_Installation.url"}; DestDir: {app}\doc; DestName: "After_Installation.url"; Components: ServerComponent DevAdminComponent; Flags: sharedfile ignoreversion Source: {#ScriptsDir}\firebirdsql.org.url; DestDir: {app}\doc; Components: ServerComponent DevAdminComponent; Flags: sharedfile ignoreversion ;Always install the original english version Source: {#GenDir}\readme.txt; DestDir: {app}; Components: DevAdminComponent; Flags: ignoreversion; @@ -543,8 +477,8 @@ Source: {#FilesDir}\fbsvcmgr.exe; DestDir: {app}; Components: DevAdminComponent; Source: {#FilesDir}\fbtracemgr.exe; DestDir: {app}; Components: DevAdminComponent; Flags: ignoreversion Source: {#FilesDir}\fbclient.dll; DestDir: {app}; Components: ClientComponent; Flags: overwritereadonly sharedfile promptifolder #if PlatformTarget == "x64" -Source: {#WOW64Dir}\fbclient.dll; DestDir: {app}\WOW64; Components: ClientComponent; Flags: overwritereadonly sharedfile promptifolder -Source: {#WOW64Dir}\instclient.exe; DestDir: {app}\WOW64; Components: ClientComponent; Flags: sharedfile ignoreversion +Source: {#WOW64Dir}\fbclient.dll; DestDir: {app}\WOW64; Components: ClientComponent; Flags: overwritereadonly sharedfile promptifolder {#SkipFileIfDevStatus} +Source: {#WOW64Dir}\instclient.exe; DestDir: {app}\WOW64; Components: ClientComponent; Flags: sharedfile ignoreversion {#SkipFileIfDevStatus} #endif Source: {#FilesDir}\icuuc??.dll; DestDir: {app}; Components: ServerComponent; Flags: sharedfile ignoreversion Source: {#FilesDir}\icuin??.dll; DestDir: {app}; Components: ServerComponent; Flags: sharedfile ignoreversion @@ -563,8 +497,8 @@ Source: {#FilesDir}\{#msvcr_filename}{#msvc_runtime_major_version}{#msvc_runtime Source: {#FilesDir}\msvcp{#msvc_runtime_major_version}{#msvc_runtime_minor_version_0}.dll; DestDir: {app}; Components: ClientComponent; Flags: sharedfile; #if PlatformTarget == "x64" ;If we are installing on x64 we need some 32-bit libraries for compatibility with 32-bit applications -Source: {#WOW64Dir}\{#msvcr_filename}{#msvc_runtime_major_version}{#msvc_runtime_minor_version_0}.dll; DestDir: {app}\WOW64; Components: ClientComponent; Flags: sharedfile; -Source: {#WOW64Dir}\msvcp{#msvc_runtime_major_version}{#msvc_runtime_minor_version_0}.dll; DestDir: {app}\WOW64; Components: ClientComponent; Flags: sharedfile; +Source: {#WOW64Dir}\{#msvcr_filename}{#msvc_runtime_major_version}{#msvc_runtime_minor_version_0}.dll; DestDir: {app}\WOW64; Components: ClientComponent; Flags: sharedfile {#SkipFileIfDevStatus}; +Source: {#WOW64Dir}\msvcp{#msvc_runtime_major_version}{#msvc_runtime_minor_version_0}.dll; DestDir: {app}\WOW64; Components: ClientComponent; Flags: sharedfile {#SkipFileIfDevStatus}; #endif #endif /* #if Int(msvc_runtime_major_version,14) >= 10 */ @@ -572,10 +506,10 @@ Source: {#WOW64Dir}\msvcp{#msvc_runtime_major_version}{#msvc_runtime_minor_versi #if PlatformTarget == "x64" ;MinVersion 0,5.0 means no version of Win9x and at least Win2k if NT O/S ;In addition, O/S must have Windows Installer 3.0. -Source: {#FilesDir}\system32\vccrt{#msvc_runtime_major_version}{#msvc_runtime_minor_version_1}_x64.msi; DestDir: {tmp}; Check: HasWI30; MinVersion: 0,5.0; Components: ClientComponent; -Source: {#WOW64Dir}\system32\vccrt{#msvc_runtime_major_version}{#msvc_runtime_minor_version_1}_Win32.msi; DestDir: {tmp}; Check: HasWI30; MinVersion: 0,5.0; Components: ClientComponent; +Source: {#FilesDir}\system32\vccrt{#msvc_runtime_major_version}{#msvc_runtime_minor_version_1}_x64.msi; DestDir: {tmp}; Check: HasWI30; MinVersion: 0,5.0; Components: ClientComponent; Flags: {#SkipFileIfDevStatus} +Source: {#WOW64Dir}\system32\vccrt{#msvc_runtime_major_version}{#msvc_runtime_minor_version_1}_Win32.msi; DestDir: {tmp}; Check: HasWI30; MinVersion: 0,5.0; Components: ClientComponent; Flags: {#SkipFileIfDevStatus} #else -Source: {#FilesDir}\system32\vccrt{#msvc_runtime_major_version}{#msvc_runtime_minor_version_1}_Win32.msi; DestDir: {tmp}; Check: HasWI30; MinVersion: 0,5.0; Components: ClientComponent; +Source: {#FilesDir}\system32\vccrt{#msvc_runtime_major_version}{#msvc_runtime_minor_version_1}_Win32.msi; DestDir: {tmp}; Check: HasWI30; MinVersion: 0,5.0; Components: ClientComponent; Flags: {#SkipFileIfDevStatus} #endif #endif @@ -592,7 +526,7 @@ Source: {#FilesDir}\intl\fbintl.dll; DestDir: {app}\intl; Components: ServerComp Source: {#FilesDir}\intl\fbintl.conf; DestDir: {app}\intl; Components: ServerComponent; Flags: onlyifdoesntexist Source: {#FilesDir}\lib\*.*; DestDir: {app}\lib; Components: DevAdminComponent; Flags: ignoreversion; #if PlatformTarget == "x64" -Source: {#WOW64Dir}\lib\*.lib; DestDir: {app}\WOW64\lib; Components: DevAdminComponent; Flags: ignoreversion +Source: {#WOW64Dir}\lib\*.lib; DestDir: {app}\WOW64\lib; Components: DevAdminComponent; Flags: ignoreversion {#SkipFileIfDevStatus} #endif ;deprecated in FB4.0 @@ -832,7 +766,7 @@ begin InstallRootDir := Default; // but the user has changed the default if (( InstallRootDir = '') and - ( FirebirdVer[0] = {#MajorVer} ) and ( FirebirdVer[1] = {#MinorVer} ) ) then // Firebird 2.n is installed + ( FirebirdVer[0] = {#FB_MAJOR_VER} ) and ( FirebirdVer[1] = {#FB_MINOR_VER} ) ) then // Firebird 2.n is installed InstallRootDir := FirebirdRootDir; // but the user has changed the default // if we haven't found anything then try the FIREBIRD env var diff --git a/builds/install/arch-specific/win32/Readme_DEV.txt b/builds/install/arch-specific/win32/Readme_DEV.txt index f245899119..5b2568ca1a 100644 --- a/builds/install/arch-specific/win32/Readme_DEV.txt +++ b/builds/install/arch-specific/win32/Readme_DEV.txt @@ -28,6 +28,8 @@ considered ready for use in production. o Please make sure you read the installation readme and the release notes. + + Reporting Bugs ============== diff --git a/builds/install/arch-specific/win32/i18n_readme.txt b/builds/install/arch-specific/win32/i18n_readme.txt index 541e9460c0..d4f4be669f 100644 --- a/builds/install/arch-specific/win32/i18n_readme.txt +++ b/builds/install/arch-specific/win32/i18n_readme.txt @@ -9,7 +9,7 @@ documentatation i18n should be available separately. I18n is a good thing, but bloating the installer with large amounts of translated documentation is not desirable. -The current version of InnoSetup used by Firebird $MAJOR.$MINOR - 5.5.8 - provides +The current version of InnoSetup used by Firebird - 5.5.8 - provides generic support for the following languages: BrazilianPortuguese, Catalan, Corsican, Czech, Danish, Dutch, Finnish, French, diff --git a/builds/install/misc/firebird.conf.in b/builds/install/misc/firebird.conf.in index 5830d25a9e..f627cd464e 100644 --- a/builds/install/misc/firebird.conf.in +++ b/builds/install/misc/firebird.conf.in @@ -180,7 +180,7 @@ # the same restrictions as in previous FB versions. To specify access # to specific trees, enum all required paths (for Windows this may be # something like 'C:\ExternalFunctions', for unix - '/db/udf;/mnt/udf'). -@UDF_COMMENT@ +# # NOTE: THE EXTERNAL FUNCTION ENGINE FEATURE COULD BE USED TO COMPROMISE # THE SERVER/HOST AS WELL AS DATABASE SECURITY!! # diff --git a/builds/win32/make_all.bat b/builds/win32/make_all.bat index d1f6ec1b7f..a7e2cbe8d2 100644 --- a/builds/win32/make_all.bat +++ b/builds/win32/make_all.bat @@ -64,8 +64,7 @@ for %%v in (gpre_boot build_msg codes) do ( :: Firebird.conf, etc @copy %FB_GEN_DIR%\firebird.msg %FB_OUTPUT_DIR% > nul -:: The line @UDF_COMMENT@ should be deleted from the target file. -findstr /V "@UDF_COMMENT@" %FB_ROOT_PATH%\builds\install\misc\firebird.conf.in > %FB_OUTPUT_DIR%\firebird.conf +@copy %FB_ROOT_PATH%\builds\install\misc\firebird.conf.in %FB_OUTPUT_DIR%\firebird.conf @copy %FB_ROOT_PATH%\builds\install\misc\databases.conf.in %FB_OUTPUT_DIR%\databases.conf >nul @copy %FB_ROOT_PATH%\builds\install\misc\fbintl.conf %FB_OUTPUT_DIR%\intl >nul @copy %FB_ROOT_PATH%\builds\install\misc\plugins.conf %FB_OUTPUT_DIR% >nul diff --git a/builds/win32/make_boot.bat b/builds/win32/make_boot.bat index 5683e08830..dd00e6e3bd 100644 --- a/builds/win32/make_boot.bat +++ b/builds/win32/make_boot.bat @@ -75,7 +75,7 @@ if "%ERRLEV%"=="1" goto :END call :isql if "%ERRLEV%"=="1" goto :END -@findstr /V "@UDF_COMMENT@" %FB_ROOT_PATH%\builds\install\misc\firebird.conf.in > %FB_BIN_DIR%\firebird.conf +@copy %FB_ROOT_PATH%\builds\install\misc\firebird.conf.in %FB_BIN_DIR%\firebird.conf :: Copy ICU and zlib both to Debug and Release configurations diff --git a/builds/win32/make_examples.bat b/builds/win32/make_examples.bat index e3db17867f..60df8c9936 100644 --- a/builds/win32/make_examples.bat +++ b/builds/win32/make_examples.bat @@ -1,5 +1,4 @@ -::@echo off - +@echo off :: Set env vars @call setenvvar.bat @@ -38,8 +37,9 @@ if errorlevel 1 ( @call :MOVE2 @goto :EOF -::=========== + :BUILD_EMPBUILD +::=========== @echo. @echo Building empbuild.fdb @copy /y %FB_ROOT_PATH%\examples\empbuild\*.sql %FB_GEN_DIR%\examples\ > nul @@ -47,7 +47,7 @@ if errorlevel 1 ( @echo. :: Here we must use cd because isql does not have an option to set a base directory -@cd "%FB_LONG_ROOT_PATH%\gen\examples" +@pushd "%FB_LONG_ROOT_PATH%\gen\examples" @echo Creating empbuild.fdb... @echo. @del empbuild.fdb 2> nul @@ -62,7 +62,8 @@ if defined FB2_INTLEMP ( @%FB_BIN_DIR%\isql -i intlbld.sql ) -@cd "%FB_LONG_ROOT_PATH%\builds\win32" +@popd + @echo. @echo path = %FB_GEN_DB_DIR%\examples @echo Preprocessing empbuild.e... @@ -75,17 +76,18 @@ if defined FB2_INTLEMP ( @%FB_BIN_DIR%\gpre.exe -r -m -n -z %FB_ROOT_PATH%\examples\empbuild\intlbld.e %FB_GEN_DIR%\examples\intlbld.c -b %FB_GEN_DB_DIR%/examples/ ) +::End of BUILD_EMPBUILD +::--------------------- @goto :EOF -::=========== :MOVE +::=========== @echo. @rmdir /q /s %FB_OUTPUT_DIR%\examples 2>nul @mkdir %FB_OUTPUT_DIR%\examples @mkdir %FB_OUTPUT_DIR%\examples\api @mkdir %FB_OUTPUT_DIR%\examples\dbcrypt -@mkdir %FB_OUTPUT_DIR%\examples\build_unix @mkdir %FB_OUTPUT_DIR%\examples\build_win32 @mkdir %FB_OUTPUT_DIR%\examples\empbuild @mkdir %FB_OUTPUT_DIR%\examples\include @@ -97,23 +99,20 @@ if defined FB2_INTLEMP ( @mkdir %FB_OUTPUT_DIR%\plugins\udr 2>nul @echo Moving files to output directory -@copy %FB_ROOT_PATH%\examples\* %FB_OUTPUT_DIR%\examples > nul -@ren %FB_OUTPUT_DIR%\examples\readme readme.txt > nul -@copy %FB_ROOT_PATH%\examples\api\* %FB_OUTPUT_DIR%\examples\api > nul -@copy %FB_ROOT_PATH%\examples\dbcrypt\* %FB_OUTPUT_DIR%\examples\dbcrypt > nul -@copy %FB_ROOT_PATH%\examples\build_unix\* %FB_OUTPUT_DIR%\examples\build_unix > nul -@copy %FB_ROOT_PATH%\examples\build_win32\* %FB_OUTPUT_DIR%\examples\build_win32 > nul +copy %FB_ROOT_PATH%\examples\* %FB_OUTPUT_DIR%\examples > nul +ren %FB_OUTPUT_DIR%\examples\readme readme.txt > nul +copy %FB_ROOT_PATH%\examples\api\* %FB_OUTPUT_DIR%\examples\api > nul +copy %FB_ROOT_PATH%\examples\dbcrypt\* %FB_OUTPUT_DIR%\examples\dbcrypt > nul +copy %FB_ROOT_PATH%\examples\build_win32\* %FB_OUTPUT_DIR%\examples\build_win32 > nul :: @copy %FB_ROOT_PATH%\examples\empbuild\* %FB_OUTPUT_DIR%\examples\empbuild > nul -@copy %FB_ROOT_PATH%\examples\empbuild\employe2.sql %FB_OUTPUT_DIR%\examples\empbuild > nul -@copy %FB_ROOT_PATH%\examples\include\* %FB_OUTPUT_DIR%\examples\include > nul -@copy %FB_ROOT_PATH%\examples\interfaces\* %FB_OUTPUT_DIR%\examples\interfaces > nul -@copy %FB_ROOT_PATH%\examples\package\* %FB_OUTPUT_DIR%\examples\package > nul -@copy %FB_ROOT_PATH%\examples\stat\* %FB_OUTPUT_DIR%\examples\stat > nul -@copy %FB_ROOT_PATH%\examples\udf\* %FB_OUTPUT_DIR%\examples\udf > nul -@copy %FB_ROOT_PATH%\examples\udr\* %FB_OUTPUT_DIR%\examples\udr > nul -@copy %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\plugins\udr\*.dll %FB_OUTPUT_DIR%\plugins\udr >nul -@copy %FB_ROOT_PATH%\src\extlib\ib_udf* %FB_OUTPUT_DIR%\examples\udf > nul -@copy %FB_ROOT_PATH%\src\extlib\fbudf\* %FB_OUTPUT_DIR%\examples\udf > nul +copy %FB_ROOT_PATH%\examples\empbuild\employe2.sql %FB_OUTPUT_DIR%\examples\empbuild > nul +copy %FB_ROOT_PATH%\examples\include\* %FB_OUTPUT_DIR%\examples\include > nul +copy %FB_ROOT_PATH%\examples\interfaces\* %FB_OUTPUT_DIR%\examples\interfaces > nul +copy %FB_ROOT_PATH%\examples\package\* %FB_OUTPUT_DIR%\examples\package > nul +copy %FB_ROOT_PATH%\examples\stat\* %FB_OUTPUT_DIR%\examples\stat > nul +copy %FB_ROOT_PATH%\examples\udf\* %FB_OUTPUT_DIR%\examples\udf > nul +copy %FB_ROOT_PATH%\examples\udr\* %FB_OUTPUT_DIR%\examples\udr > nul +copy %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\plugins\udr\*.dll %FB_OUTPUT_DIR%\plugins\udr >nul ::@copy %FB_GEN_DIR%\examples\empbuild.c %FB_OUTPUT_DIR%\examples\empbuild\ > nul ::@copy %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\examples\empbuild.exe %FB_GEN_DIR%\examples\empbuild.exe > nul @@ -124,37 +123,50 @@ if defined FB2_INTLEMP ( ::@copy %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\examples\intlbuild.exe %FB_GEN_DIR%\examples\intlbuild.exe > nul ::) ::) + +::End of MOVE +::----------- @goto :EOF +:BUILD_EMPLOYEE ::=========== :: only to test if it works -:BUILD_EMPLOYEE + @echo. @echo Building employee.fdb -:: Here we must use cd because isql does not have an option to set a base directory -:: and empbuild.exe uses isql -@set FB_SAVE_PATH=%PATH% -@set PATH=%FB_BIN_DIR%;%PATH% + +:: Do no mess with global variables +setlocal + :: This allows us to use the new engine in embedded mode to build :: the employee database. @set FIREBIRD=%FB_BIN_DIR% +@set PATH=%FB_BIN_DIR%;%PATH% -@cd "%FB_LONG_ROOT_PATH%\gen\examples" -@del %FB_GEN_DIR%\examples\employee.fdb 2>nul -@%FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\empbuild\empbuild.exe %FB_GEN_DB_DIR%/examples/employee.fdb -if errorlevel 44 (call :ERROR empbuild.exe failed - see empbuild_%FB_TARGET_PLATFORM%.log for details & goto :EOF) +:: Here we must use cd because isql does not have an option to set a base directory +:: and empbuild.exe uses isql +:: BEWARE: It will run without error if you have FB client from previous version +:: installed in System32 and server run but created database will have +:: wrong ODS. +@pushd "%FB_GEN_DIR%\examples" +if exist employee.fdb del employee.fdb + +%FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\empbuild\empbuild.exe %FB_GEN_DB_DIR%/examples/employee.fdb +if errorlevel 44 (call :ERROR empbuild.exe failed - see empbuild_%FB_TARGET_PLATFORM%.log for details ) @if defined FB2_INTLEMP ( @echo Building intlemp.fdb @del %FB_GEN_DIR%\examples\intlemp.fdb 2>nul @del isql.tmp 2>nul - @echo s;intlemp.fdb;%SERVER_NAME%:%FB_GEN_DIR%\examples\intlemp.fdb;g > isql.tmp + @echo s;intlemp.fdb;%FB_GEN_DIR%\examples\intlemp.fdb;g > isql.tmp @%FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\intlbuild\intlbuild.exe %FB_GEN_DB_DIR%/examples/intlemp.fdb ) -@set FIREBIRD= -@set PATH=%FB_SAVE_PATH% -@cd "%FB_LONG_ROOT_PATH%\builds\win32" +@popd +endlocal + +::End of BUILD_EMPLOYEE +::--------------------- @goto :EOF ::============== @@ -182,8 +194,8 @@ if defined FB2_INTLEMP ( @echo Error - %* @echo. set ERRLEV=1 -cancel_script > nul 2>&1 + +exit /b 1 + ::End of ERROR ::------------ -@goto :EOF - diff --git a/builds/win32/setenvvar.bat b/builds/win32/setenvvar.bat index ce2bbfec59..724f8bfc26 100644 --- a/builds/win32/setenvvar.bat +++ b/builds/win32/setenvvar.bat @@ -4,11 +4,6 @@ :: FB_DB_PATH unix format path of the main directory :: (This is used by gpre and preprocess.bat) :: VS_VER VisualStudio version (msvc10|msvc12|msvc14) -:: SERVER_NAME server needed to connect to firebird (could include port) -:: Example : localhost/3051 -:: (Note - SERVER_NAME is almost deprecated - it is only used by -:: make_examples.bat -:: @echo off @@ -48,8 +43,6 @@ set VS_VER=msvc%MSVC_VERSION% ::================= :SET_DB_DIR -@SET SERVER_NAME=localhost - @cd ..\.. @for /f "delims=" %%a in ('@cd') do (set FB_LONG_ROOT_PATH=%%a) @for /f "delims=" %%a in ('@cd') do (set FB_ROOT_PATH=%%~sa) @@ -108,7 +101,6 @@ if defined VS_VER_EXPRESS ( @echo msvc_version=%MSVC_VERSION% @echo db_path=%FB_DB_PATH% @echo root_path=%FB_ROOT_PATH% -@echo server_name=%SERVER_NAME% @echo. @echo (End of %0) @echo. diff --git a/configure.ac b/configure.ac index ac5a445f82..8dead2da79 100644 --- a/configure.ac +++ b/configure.ac @@ -1371,9 +1371,6 @@ else fi AC_SUBST(POSTFIX_INCLUDE) -UDF_COMMENT="#" -AC_SUBST(UDF_COMMENT) - AC_CONFIG_FILES( gen/make.rules:${MAKE_SRC_DIR}/make.rules gen/make.defaults:${MAKE_SRC_DIR}/make.defaults diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dfedabe21b..b46b91ec12 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -909,7 +909,7 @@ add_custom_target(copy_files COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GENERATED_DIR}/security.fdb ${output_dir}/security4.fdb COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GENERATED_DIR}/help.fdb ${output_dir}/help/help.fdb # configs, text files - COMMAND sed "/@UDF_COMMENT@/d" < ${CMAKE_SOURCE_DIR}/builds/install/misc/firebird.conf.in > ${output_dir}/firebird.conf + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/builds/install/misc/firebird.conf.in ${output_dir}/firebird.conf COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/builds/install/misc/databases.conf.in ${output_dir}/databases.conf COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/builds/install/misc/fbintl.conf ${output_dir}/intl/fbintl.conf COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/builds/install/misc/plugins.conf ${output_dir}/plugins.conf From b9f4c652202eadefc2b5267d139545f5d6878043 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Wed, 15 Jan 2020 18:02:03 +0300 Subject: [PATCH 219/274] Added various ways to get infomation about wire crypt plugin name: variable in rdb$get_context(), field in mon$attachments and tag in Attachment::getInfo() --- doc/README.monitoring_tables | 1 + doc/sql.extensions/README.context_variables2 | 3 +++ doc/sql.extensions/README.isc_info_xxx | 14 +++++++++++++- src/include/firebird/impl/consts_pub.h | 11 +++++++++-- src/include/firebird/impl/inf_pub.h | 2 ++ src/include/gen/ids.h | 1 + src/isql/show.epp | 6 ++++++ src/jrd/Attachment.cpp | 1 + src/jrd/Attachment.h | 1 + src/jrd/Monitoring.cpp | 2 ++ src/jrd/SysFunction.cpp | 8 ++++++++ src/jrd/fields.h | 3 ++- src/jrd/inf.cpp | 13 +++++++++++++ src/jrd/jrd.cpp | 5 +++++ src/jrd/names.h | 1 + src/jrd/relations.h | 1 + src/remote/remote.h | 5 +++-- src/remote/server/server.cpp | 4 ++++ 18 files changed, 76 insertions(+), 6 deletions(-) diff --git a/doc/README.monitoring_tables b/doc/README.monitoring_tables index b6918f661d..ea5477651c 100644 --- a/doc/README.monitoring_tables +++ b/doc/README.monitoring_tables @@ -114,6 +114,7 @@ Monitoring tables - MON$STATEMENT_TIMEOUT (statement timeout) - MON$WIRE_COMPRESSED (wire compression enabled/disabled) - MON$WIRE_ENCRYPTED (wire encryption enabled/disabled) + - MON$WIRE_CRYPT_PLUGIN (name of wire encryption plugin) MON$TRANSACTIONS (started transactions) - MON$TRANSACTION_ID (transaction ID) diff --git a/doc/sql.extensions/README.context_variables2 b/doc/sql.extensions/README.context_variables2 index bac0d8ec92..1d6ef72ab9 100644 --- a/doc/sql.extensions/README.context_variables2 +++ b/doc/sql.extensions/README.context_variables2 @@ -64,6 +64,9 @@ Usage: WIRE_ENCRYPTED | Encryption status of current connection. | Value is the same as for compression status above. | + WIRE_CRYPT_PLUGIN | If connection is encrypted - returns name of current plugin, + | otherwise NULL. + | CLIENT_ADDRESS | The wire protocol address and port number of remote client | represented as string. Value is IP address concatenated with | port number using the '/' separator character. Value is diff --git a/doc/sql.extensions/README.isc_info_xxx b/doc/sql.extensions/README.isc_info_xxx index 7682f29597..e6aa9d0740 100644 --- a/doc/sql.extensions/README.isc_info_xxx +++ b/doc/sql.extensions/README.isc_info_xxx @@ -32,10 +32,22 @@ New items for isc_database_info See also CORE-2054. +4. Database encryption information: + fb_info_crypt_state - flags describing encryption state: + fb_info_crypt_encrypted - database is encrypted, + fb_info_crypt_process - encryption/decryption process is not complete; + fb_info_crypt_plugin - name of database crypt plugin; + fb_info_crypt_key - name of used database crypt key. + +5. Connection information: + fb_info_conn_flags - flags describing connection state: + isc_dpb_addr_flag_conn_compressed - compression is used for connection, + isc_dpb_addr_flag_conn_encrypted - connection is encrypted; + fb_info_wire_crypt - name of connection encryption plugin. New items for isc_transaction_info: - + 1. isc_info_tra_oldest_interesting : return number of oldest interesting transaction when current transaction started. For snapshot transactions this is also the diff --git a/src/include/firebird/impl/consts_pub.h b/src/include/firebird/impl/consts_pub.h index fc6f912f92..9ba14d70f9 100644 --- a/src/include/firebird/impl/consts_pub.h +++ b/src/include/firebird/impl/consts_pub.h @@ -153,8 +153,9 @@ ::= isc_dpb_addr_protocol | - isc_dpb_addr_endpoint - isc_dpb_addr_flags + isc_dpb_addr_endpoint | + isc_dpb_addr_flags | + isc_dpb_addr_crypt ::= "TCPv4" | @@ -163,6 +164,11 @@ "WNET" | .... + ::= + "Arc4" | + "ChaCha" | + .... + ::= | // such as "172.20.1.1" | // such as "2001:0:13FF:09FF::1" @@ -178,6 +184,7 @@ #define isc_dpb_addr_protocol 1 #define isc_dpb_addr_endpoint 2 #define isc_dpb_addr_flags 3 +#define isc_dpb_addr_crypt 4 /* possible addr flags */ #define isc_dpb_addr_flag_conn_compressed 0x01 diff --git a/src/include/firebird/impl/inf_pub.h b/src/include/firebird/impl/inf_pub.h index ea4d31eb9d..003ef9a68d 100644 --- a/src/include/firebird/impl/inf_pub.h +++ b/src/include/firebird/impl/inf_pub.h @@ -160,6 +160,8 @@ enum db_info_types fb_info_creation_timestamp_tz = 139, + fb_info_wire_crypt = 140, + isc_info_db_last_value /* Leave this LAST! */ }; diff --git a/src/include/gen/ids.h b/src/include/gen/ids.h index 9c3beca6b8..31cf55039c 100644 --- a/src/include/gen/ids.h +++ b/src/include/gen/ids.h @@ -524,6 +524,7 @@ const USHORT f_mon_att_stmt_timeout = 22; const USHORT f_mon_att_wire_compressed = 23; const USHORT f_mon_att_wire_encrypted = 24; + const USHORT f_mon_att_remote_crypt = 25; // Relation 35 (MON$TRANSACTIONS) diff --git a/src/isql/show.epp b/src/isql/show.epp index b36ebeb6ab..cdd23a5ea8 100644 --- a/src/isql/show.epp +++ b/src/isql/show.epp @@ -251,6 +251,7 @@ static const UCHAR db_items[] = isc_info_db_id, #endif fb_info_crypt_state, + fb_info_wire_crypt, isc_info_end }; @@ -554,6 +555,11 @@ bool SHOW_dbb_parameters(Firebird::IAttachment* db_handle, (SLONG) isqlGlob.major_ods, value_out, separator); break; + case fb_info_wire_crypt: + if (d) + sprintf (info, "Wire crypt plugin: %.*s%s", length, d, separator); + break; + #ifdef DEV_BUILD case isc_info_db_id: { diff --git a/src/jrd/Attachment.cpp b/src/jrd/Attachment.cpp index b4d32104f0..b380b6e748 100644 --- a/src/jrd/Attachment.cpp +++ b/src/jrd/Attachment.cpp @@ -221,6 +221,7 @@ Jrd::Attachment::Attachment(MemoryPool* pool, Database* dbb) att_context_vars(*pool), ddlTriggersContext(*pool), att_network_protocol(*pool), + att_remote_crypt(*pool), att_remote_address(*pool), att_remote_process(*pool), att_client_version(*pool), diff --git a/src/jrd/Attachment.h b/src/jrd/Attachment.h index 847e45c00b..93769141f7 100644 --- a/src/jrd/Attachment.h +++ b/src/jrd/Attachment.h @@ -440,6 +440,7 @@ public: Firebird::StringMap att_context_vars; // Context variables for the connection Firebird::Stack ddlTriggersContext; // Context variables for DDL trigger event Firebird::string att_network_protocol; // Network protocol used by client for connection + Firebird::PathName att_remote_crypt; // Name of wire crypt plugin (if any) Firebird::string att_remote_address; // Protocol-specific address of remote client SLONG att_remote_pid; // Process id of remote client ULONG att_remote_flags; // Flags specific for server/client link diff --git a/src/jrd/Monitoring.cpp b/src/jrd/Monitoring.cpp index 7a9986cf9a..49a02453e9 100644 --- a/src/jrd/Monitoring.cpp +++ b/src/jrd/Monitoring.cpp @@ -978,6 +978,8 @@ void Monitoring::putAttachment(SnapshotData::DumpRecord& record, const Jrd::Atta record.storeString(f_mon_att_client_version, attachment->att_client_version); // remote protocol version record.storeString(f_mon_att_remote_version, attachment->att_remote_protocol); + // wire encryption plugin + record.storeString(f_mon_att_remote_crypt, attachment->att_remote_crypt); // remote host name record.storeString(f_mon_att_remote_host, attachment->att_remote_host); // OS user name diff --git a/src/jrd/SysFunction.cpp b/src/jrd/SysFunction.cpp index 9273c840a1..be5e09b45d 100644 --- a/src/jrd/SysFunction.cpp +++ b/src/jrd/SysFunction.cpp @@ -335,6 +335,7 @@ const char NETWORK_PROTOCOL_NAME[] = "NETWORK_PROTOCOL", WIRE_COMPRESSED_NAME[] = "WIRE_COMPRESSED", WIRE_ENCRYPTED_NAME[] = "WIRE_ENCRYPTED", + WIRE_CRYPT_PLUGIN_NAME[] = "WIRE_CRYPT_PLUGIN", CLIENT_ADDRESS_NAME[] = "CLIENT_ADDRESS", CLIENT_HOST_NAME[] = "CLIENT_HOST", CLIENT_PID_NAME[] = "CLIENT_PID", @@ -4048,6 +4049,13 @@ dsc* evlGetContext(thread_db* tdbb, const SysFunction*, const NestValueArray& ar resultStr = (attachment->att_remote_flags & isc_dpb_addr_flag_conn_encrypted) ? TRUE_VALUE : FALSE_VALUE; } + else if (nameStr == WIRE_CRYPT_PLUGIN_NAME) + { + if (attachment->att_remote_crypt.isEmpty()) + return NULL; + + resultStr = attachment->att_remote_crypt.ToString(); + } else if (nameStr == CLIENT_ADDRESS_NAME) { if (attachment->att_remote_address.isEmpty()) diff --git a/src/jrd/fields.h b/src/jrd/fields.h index 36f75f9497..aa60900fa4 100644 --- a/src/jrd/fields.h +++ b/src/jrd/fields.h @@ -208,4 +208,5 @@ FIELD(fld_tz_db_version , nam_tz_db_version , dtype_varying , 10 , dsc_text_type_ascii , NULL , true) - FIELD(fld_crypt_state , nam_crypt_state , dtype_short , sizeof(SSHORT) , 0 , NULL , true) \ No newline at end of file + FIELD(fld_crypt_state , nam_crypt_state , dtype_short , sizeof(SSHORT) , 0 , NULL , true) + FIELD(fld_remote_crypt , nam_wire_crypt_plugin, dtype_varying, MAX_SQL_IDENTIFIER_LEN , dsc_text_type_metadata , NULL , true) diff --git a/src/jrd/inf.cpp b/src/jrd/inf.cpp index 27babdf081..38b1a5a742 100644 --- a/src/jrd/inf.cpp +++ b/src/jrd/inf.cpp @@ -822,6 +822,19 @@ void INF_database_info(thread_db* tdbb, length = INF_convert(tdbb->getAttachment()->att_remote_flags, buffer); break; + case fb_info_wire_crypt: + { + const PathName& nm = tdbb->getAttachment()->att_remote_crypt; + if (!(info = INF_put_item(item, static_cast(nm.length()), nm.c_str(), info, end))) + { + if (transaction) + TRA_commit(tdbb, transaction, false); + + return; + } + } + continue; + case fb_info_statement_timeout_db: length = INF_convert(dbb->dbb_config->getStatementTimeout(), buffer); break; diff --git a/src/jrd/jrd.cpp b/src/jrd/jrd.cpp index 0a53d15f38..bc82593032 100644 --- a/src/jrd/jrd.cpp +++ b/src/jrd/jrd.cpp @@ -1065,6 +1065,7 @@ namespace Jrd PathName dpb_working_directory; string dpb_set_db_charset; string dpb_network_protocol; + PathName dpb_remote_crypt; string dpb_remote_address; string dpb_remote_host; string dpb_remote_os_user; @@ -6970,6 +6971,9 @@ void DatabaseOptions::get(const UCHAR* dpb, USHORT dpb_length, bool& invalid_cli case isc_dpb_addr_flags: dpb_remote_flags = address.getInt(); break; + case isc_dpb_addr_crypt: + address.getPath(dpb_remote_crypt); + break; default: break; } @@ -7275,6 +7279,7 @@ static JAttachment* create_attachment(const PathName& alias_name, attachment->att_filename = alias_name; attachment->att_network_protocol = options.dpb_network_protocol; + attachment->att_remote_crypt = options.dpb_remote_crypt; attachment->att_remote_address = options.dpb_remote_address; attachment->att_remote_pid = options.dpb_remote_pid; attachment->att_remote_flags = options.dpb_remote_flags; diff --git a/src/jrd/names.h b/src/jrd/names.h index 05b405cd47..6ebe4528c0 100644 --- a/src/jrd/names.h +++ b/src/jrd/names.h @@ -419,6 +419,7 @@ NAME("MON$STATEMENT_TIMER", nam_stmt_timer) NAME("MON$WIRE_COMPRESSED", nam_wire_compressed) NAME("MON$WIRE_ENCRYPTED", nam_wire_encrypted) +NAME("MON$WIRE_CRYPT_PLUGIN", nam_wire_crypt_plugin) NAME("RDB$TIME_ZONES", nam_time_zones) NAME("RDB$TIME_ZONE_ID", nam_tz_id) diff --git a/src/jrd/relations.h b/src/jrd/relations.h index 37ce864eb8..ceb93f0c6c 100644 --- a/src/jrd/relations.h +++ b/src/jrd/relations.h @@ -523,6 +523,7 @@ RELATION(nam_mon_attachments, rel_mon_attachments, ODS_11_1, rel_virtual) FIELD(f_mon_att_stmt_timeout, nam_stmt_timeout, fld_stmt_timeout, 0, ODS_13_0) FIELD(f_mon_att_wire_compressed, nam_wire_compressed, fld_bool, 0, ODS_13_0) FIELD(f_mon_att_wire_encrypted, nam_wire_encrypted, fld_bool, 0, ODS_13_0) + FIELD(f_mon_att_remote_crypt, nam_wire_crypt_plugin, fld_remote_crypt, 0, ODS_12_0) END_RELATION // Relation 35 (MON$TRANSACTIONS) diff --git a/src/remote/remote.h b/src/remote/remote.h index 6e575106fb..8ffa2f8ced 100644 --- a/src/remote/remote.h +++ b/src/remote/remote.h @@ -1093,6 +1093,7 @@ struct rem_port : public Firebird::GlobalStorage, public Firebird::RefCounted Firebird::IWireCryptPlugin* port_crypt_plugin; // plugin used by port, when not NULL - crypts wire data Firebird::ICryptKeyCallback* port_client_crypt_callback; // client callback to transfer database crypt key ServerCallbackBase* port_server_crypt_callback; // server callback to transfer database crypt key + Firebird::PathName port_crypt_name; // name of actual wire crypt plugin Firebird::RefPtr port_replicator; @@ -1138,8 +1139,8 @@ public: port_srv_auth(NULL), port_srv_auth_block(NULL), port_crypt_keys(getPool()), port_crypt_complete(false), port_crypt_level(WIRECRYPT_REQUIRED), port_known_server_keys(getPool()), port_crypt_plugin(NULL), - port_client_crypt_callback(NULL), port_server_crypt_callback(NULL), port_replicator(NULL), - port_buffer(FB_NEW_POOL(getPool()) UCHAR[rpt]), + port_client_crypt_callback(NULL), port_server_crypt_callback(NULL), port_crypt_name(getPool()), + port_replicator(NULL), port_buffer(FB_NEW_POOL(getPool()) UCHAR[rpt]), port_snd_packets(0), port_rcv_packets(0), port_snd_bytes(0), port_rcv_bytes(0) { addRef(); diff --git a/src/remote/server/server.cpp b/src/remote/server/server.cpp index 2a9c1a3825..425deb35e5 100644 --- a/src/remote/server/server.cpp +++ b/src/remote/server/server.cpp @@ -2328,7 +2328,10 @@ static void addClumplets(ClumpletWriter* dpb_buffer, flags |= isc_dpb_addr_flag_conn_compressed; #endif if (port->port_crypt_plugin) + { flags |= isc_dpb_addr_flag_conn_encrypted; + address_record.insertString(isc_dpb_addr_crypt, port->port_crypt_name); + } if (flags) address_record.insertInt(isc_dpb_addr_flags, flags); @@ -6175,6 +6178,7 @@ void rem_port::start_crypt(P_CRYPT * crypt, PACKET* sendL) port_crypt_plugin = cp.plugin(); port_crypt_plugin->addRef(); port_crypt_complete = true; + port_crypt_name = cp.name(); send_response(sendL, 0, 0, &st, false); WIRECRYPT_DEBUG(fprintf(stderr, "Srv: Installed cipher %s\n", cp.name())); From 236fc7ac14fc67ab779e759d2ee030ea4822de3b Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Tue, 14 Jan 2020 13:07:47 -0300 Subject: [PATCH 220/274] Added github action cronjob for automatic creation of tzdata update pull request. --- .github/workflows/tzdata-update.yml | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/tzdata-update.yml diff --git a/.github/workflows/tzdata-update.yml b/.github/workflows/tzdata-update.yml new file mode 100644 index 0000000000..7af28a4b59 --- /dev/null +++ b/.github/workflows/tzdata-update.yml @@ -0,0 +1,37 @@ +name: tzdata-update + +on: + schedule: + - cron: '0 11 * * *' + +jobs: + tzdata-update: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Checkout ICU + run: git clone --depth 1 https://github.com/unicode-org/icu-data.git -b master /tmp/icu-checkout + + - name: Check and update + run: | + VERSION=`ls /tmp/icu-checkout/tzdata/icunew/ -r1a |head -1` + echo Last version: $VERSION + + if [ "$VERSION" == "`cat tzdata/version.txt`" ] + then + exit + fi + + echo $VERSION > tzdata/version.txt + tzdata/update.sh + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: Updata tzdata. + title: Updata tzdata. + assignees: asfernandes + branch: work/tzdata-update From 98fe361f98ce130236681ea271a1b0402f7cbaac Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Thu, 16 Jan 2020 00:04:55 +0000 Subject: [PATCH 221/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 7c1443a4dd..9be0f3f4e8 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1726 + FORMAL BUILD NUMBER:1729 */ -#define PRODUCT_VER_STRING "4.0.0.1726" -#define FILE_VER_STRING "WI-T4.0.0.1726" -#define LICENSE_VER_STRING "WI-T4.0.0.1726" -#define FILE_VER_NUMBER 4, 0, 0, 1726 +#define PRODUCT_VER_STRING "4.0.0.1729" +#define FILE_VER_STRING "WI-T4.0.0.1729" +#define LICENSE_VER_STRING "WI-T4.0.0.1729" +#define FILE_VER_NUMBER 4, 0, 0, 1729 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1726" +#define FB_BUILD_NO "1729" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 97e03e1981..a7ebb3415b 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1726 +BuildNum=1729 NowAt=`pwd` cd `dirname $0` From 55ef2beb209e84ab3ccf4e6a20a0ea7dd185d39e Mon Sep 17 00:00:00 2001 From: Dimitry Sibiryakov Date: Wed, 15 Jan 2020 18:01:32 +0100 Subject: [PATCH 222/274] firebird.conf and databases.conf without .in suffix --- builds/install/arch-specific/solaris/CS/prototype.in | 2 +- builds/install/arch-specific/solaris/SS/prototype.in | 2 +- .../arch-specific/win32/BuildExecutableInstall.bat | 2 +- builds/install/misc/{databases.conf.in => databases.conf} | 0 builds/install/misc/{firebird.conf.in => firebird.conf} | 0 builds/win32/make_all.bat | 8 ++++---- builds/win32/make_boot.bat | 2 +- configure.ac | 4 ++-- src/CMakeLists.txt | 4 ++-- src/jrd/constants.h | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) rename builds/install/misc/{databases.conf.in => databases.conf} (100%) rename builds/install/misc/{firebird.conf.in => firebird.conf} (100%) diff --git a/builds/install/arch-specific/solaris/CS/prototype.in b/builds/install/arch-specific/solaris/CS/prototype.in index 1443a4748a..360c6bc1cf 100644 --- a/builds/install/arch-specific/solaris/CS/prototype.in +++ b/builds/install/arch-specific/solaris/CS/prototype.in @@ -18,7 +18,7 @@ d none @prefix@ 0751 firebird firebird v CONFIG.prsv @prefix@/security2.fdb=$SRCDIR/security2.fdb 0660 firebird firebird v CONFIG.prsv @prefix@/firebird.conf=$SRCDIR/misc/firebird.conf 0444 firebird firebird -v CONFIG.prsv @prefix@/databases.conf=$SRCDIR/../../builds/install/misc/databases.conf.in 0444 firebird firebird +v CONFIG.prsv @prefix@/databases.conf=$SRCDIR/../../builds/install/misc/databases.conf 0444 firebird firebird v CONFIG.prsv @prefix@/intl/fbintl.conf=$SRCDIR/misc/fbintl.conf 0644 firebird firebird f none @prefix@/CHANGELOG.md=$SRCDIR/../../CHANGELOG.md 0644 firebird firebird f none @prefix@/README.md=$SRCDIR/../../README.md 0644 firebird firebird diff --git a/builds/install/arch-specific/solaris/SS/prototype.in b/builds/install/arch-specific/solaris/SS/prototype.in index bb928b98a3..35bd67adb8 100644 --- a/builds/install/arch-specific/solaris/SS/prototype.in +++ b/builds/install/arch-specific/solaris/SS/prototype.in @@ -18,7 +18,7 @@ d none @prefix@ 0755 firebird firebird v CONFIG.prsv @prefix@/security2.fdb=$SRCDIR/security2.fdb 0660 firebird firebird v CONFIG.prsv @prefix@/firebird.conf=$SRCDIR/misc/firebird.conf 0644 firebird firebird -v CONFIG.prsv @prefix@/databases.conf=$SRCDIR/../../builds/install/misc/databases.conf.in 0544 firebird firebird +v CONFIG.prsv @prefix@/databases.conf=$SRCDIR/../../builds/install/misc/databases.conf 0544 firebird firebird v CONFIG.prsv @prefix@/intl/fbintl.conf=$SRCDIR/misc/fbintl.conf 0644 firebird firebird f none @prefix@/CHANGELOG.md=$SRCDIR/../../CHANGELOG.md 0644 firebird firebird f none @prefix@/README.md=$SRCDIR/../../README.md 0644 firebird firebird diff --git a/builds/install/arch-specific/win32/BuildExecutableInstall.bat b/builds/install/arch-specific/win32/BuildExecutableInstall.bat index 2c0ba91ce9..095c01c752 100644 --- a/builds/install/arch-specific/win32/BuildExecutableInstall.bat +++ b/builds/install/arch-specific/win32/BuildExecutableInstall.bat @@ -406,7 +406,7 @@ endlocal :: Generate sample databases file ::=============================== @echo Creating sample databases.conf -copy %FB_ROOT_PATH%\builds\install\misc\databases.conf.in %FB_OUTPUT_DIR%\databases.conf > nul +copy %FB_ROOT_PATH%\builds\install\misc\databases.conf %FB_OUTPUT_DIR%\databases.conf > nul ::End of DB_CONF ::----------------- diff --git a/builds/install/misc/databases.conf.in b/builds/install/misc/databases.conf similarity index 100% rename from builds/install/misc/databases.conf.in rename to builds/install/misc/databases.conf diff --git a/builds/install/misc/firebird.conf.in b/builds/install/misc/firebird.conf similarity index 100% rename from builds/install/misc/firebird.conf.in rename to builds/install/misc/firebird.conf diff --git a/builds/win32/make_all.bat b/builds/win32/make_all.bat index a7e2cbe8d2..698ae1bd50 100644 --- a/builds/win32/make_all.bat +++ b/builds/win32/make_all.bat @@ -63,10 +63,10 @@ for %%v in (gpre_boot build_msg codes) do ( ) :: Firebird.conf, etc -@copy %FB_GEN_DIR%\firebird.msg %FB_OUTPUT_DIR% > nul -@copy %FB_ROOT_PATH%\builds\install\misc\firebird.conf.in %FB_OUTPUT_DIR%\firebird.conf -@copy %FB_ROOT_PATH%\builds\install\misc\databases.conf.in %FB_OUTPUT_DIR%\databases.conf >nul -@copy %FB_ROOT_PATH%\builds\install\misc\fbintl.conf %FB_OUTPUT_DIR%\intl >nul +@copy %FB_GEN_DIR%\firebird.msg %FB_OUTPUT_DIR%\ > nul +@copy %FB_ROOT_PATH%\builds\install\misc\firebird.conf %FB_OUTPUT_DIR%\firebird.conf >nul +@copy %FB_ROOT_PATH%\builds\install\misc\databases.conf %FB_OUTPUT_DIR%\databases.conf >nul +@copy %FB_ROOT_PATH%\builds\install\misc\fbintl.conf %FB_OUTPUT_DIR%\intl\ >nul @copy %FB_ROOT_PATH%\builds\install\misc\plugins.conf %FB_OUTPUT_DIR% >nul @copy %FB_ROOT_PATH%\builds\install\misc\replication.conf %FB_OUTPUT_DIR% >nul @copy %FB_ROOT_PATH%\src\utilities\ntrace\fbtrace.conf %FB_OUTPUT_DIR% >nul diff --git a/builds/win32/make_boot.bat b/builds/win32/make_boot.bat index dd00e6e3bd..d7c77277d1 100644 --- a/builds/win32/make_boot.bat +++ b/builds/win32/make_boot.bat @@ -75,7 +75,7 @@ if "%ERRLEV%"=="1" goto :END call :isql if "%ERRLEV%"=="1" goto :END -@copy %FB_ROOT_PATH%\builds\install\misc\firebird.conf.in %FB_BIN_DIR%\firebird.conf +@copy %FB_ROOT_PATH%\builds\install\misc\firebird.conf %FB_BIN_DIR%\firebird.conf :: Copy ICU and zlib both to Debug and Release configurations diff --git a/configure.ac b/configure.ac index 8dead2da79..c0e1398f62 100644 --- a/configure.ac +++ b/configure.ac @@ -1251,8 +1251,8 @@ dnl common files for all posix hosts dnl TODO: fix "arch-specific/linux/" paths for common posix scripts with SVN for fb_tgt in $FB_TARGETS; do AC_CONFIG_FILES([ -gen/$fb_tgt/firebird/firebird.conf:builds/install/misc/firebird.conf.in -gen/$fb_tgt/firebird/databases.conf:builds/install/misc/databases.conf.in +gen/$fb_tgt/firebird/firebird.conf:builds/install/misc/firebird.conf +gen/$fb_tgt/firebird/databases.conf:builds/install/misc/databases.conf gen/$fb_tgt/firebird/fbtrace.conf:src/utilities/ntrace/fbtrace.conf gen/$fb_tgt/firebird/intl/fbintl.conf:builds/install/misc/fbintl.conf gen/$fb_tgt/firebird/plugins.conf:builds/install/misc/plugins.conf diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b46b91ec12..b890c4ea8b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -909,8 +909,8 @@ add_custom_target(copy_files COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GENERATED_DIR}/security.fdb ${output_dir}/security4.fdb COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GENERATED_DIR}/help.fdb ${output_dir}/help/help.fdb # configs, text files - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/builds/install/misc/firebird.conf.in ${output_dir}/firebird.conf - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/builds/install/misc/databases.conf.in ${output_dir}/databases.conf + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/builds/install/misc/firebird.conf ${output_dir}/firebird.conf + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/builds/install/misc/databases.conf ${output_dir}/databases.conf COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/builds/install/misc/fbintl.conf ${output_dir}/intl/fbintl.conf COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/builds/install/misc/plugins.conf ${output_dir}/plugins.conf COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/builds/install/misc/IPLicense.txt ${output_dir}/IPLicense.txt diff --git a/src/jrd/constants.h b/src/jrd/constants.h index 8e0268cc77..72b463455f 100644 --- a/src/jrd/constants.h +++ b/src/jrd/constants.h @@ -62,7 +62,7 @@ const int TEMP_STR_LENGTH = 128; // Metadata constants // When changing these constants, change MaxIdentifierByteLength and MaxIdentifierCharLength in -// firebird.conf.in too. +// firebird.conf too. const unsigned METADATA_IDENTIFIER_CHAR_LEN = 63; const unsigned METADATA_BYTES_PER_CHAR = 4; From 473afd3df809d64ae3b6f224b4a569e6bc4589b2 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Thu, 16 Jan 2020 17:32:43 +0300 Subject: [PATCH 223/274] An attempt to fix CORE-6218: COUNT(DISTINCT ) leads FB to crash when there are duplicate values of this field --- src/jrd/sort.cpp | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/jrd/sort.cpp b/src/jrd/sort.cpp index f893fb044d..f44bc99bd3 100644 --- a/src/jrd/sort.cpp +++ b/src/jrd/sort.cpp @@ -819,15 +819,14 @@ void Sort::diddleKey(UCHAR* record, bool direction, bool duplicateHandling) break; case SKD_dec64: - if (direction && !duplicateHandling) + fb_assert(false); // diddleKey for Dec64/128 not tested on bigendians! + if (direction) { ((Decimal64*) p)->makeKey(lwp); *p ^= 1 << 7; } else if (!(key->skd_flags & SKD_separate_data)) { - fb_assert(false); - if (complement && n) { UCHAR* pp = p; @@ -843,15 +842,13 @@ void Sort::diddleKey(UCHAR* record, bool direction, bool duplicateHandling) case SKD_dec128: fb_assert(false); // diddleKey for Dec64/128 not tested on bigendians! - if (direction && !duplicateHandling) + if (direction) { ((Decimal128*) p)->makeKey(lwp); *p ^= 1 << 7; } else if (!(key->skd_flags & SKD_separate_data)) { - fb_assert(false); - if (complement && n) { UCHAR* pp = p; @@ -1133,15 +1130,13 @@ void Sort::diddleKey(UCHAR* record, bool direction, bool duplicateHandling) #endif // IEEE case SKD_dec64: - if (direction && !duplicateHandling) + if (direction) { ((Decimal64*) p)->makeKey(lwp); p[3] ^= 1 << 7; } else if (!(key->skd_flags & SKD_separate_data)) { - fb_assert(false); - if (complement && n) { UCHAR* pp = p; @@ -1156,15 +1151,13 @@ void Sort::diddleKey(UCHAR* record, bool direction, bool duplicateHandling) break; case SKD_dec128: - if (direction && !duplicateHandling) + if (direction) { ((Decimal128*) p)->makeKey(lwp); p[3] ^= 1 << 7; } else if (!(key->skd_flags & SKD_separate_data)) { - fb_assert(false); - if (complement && n) { UCHAR* pp = p; From 9720b5d10c8f691afab1a96a463dfa881b7b9e9c Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Fri, 17 Jan 2020 00:04:36 +0000 Subject: [PATCH 224/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 9be0f3f4e8..840d7d5a09 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1729 + FORMAL BUILD NUMBER:1731 */ -#define PRODUCT_VER_STRING "4.0.0.1729" -#define FILE_VER_STRING "WI-T4.0.0.1729" -#define LICENSE_VER_STRING "WI-T4.0.0.1729" -#define FILE_VER_NUMBER 4, 0, 0, 1729 +#define PRODUCT_VER_STRING "4.0.0.1731" +#define FILE_VER_STRING "WI-T4.0.0.1731" +#define LICENSE_VER_STRING "WI-T4.0.0.1731" +#define FILE_VER_NUMBER 4, 0, 0, 1731 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1729" +#define FB_BUILD_NO "1731" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index a7ebb3415b..263b7217af 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1729 +BuildNum=1731 NowAt=`pwd` cd `dirname $0` From be2d7718c8964af23c798a7431a7764167e5266e Mon Sep 17 00:00:00 2001 From: hvlad Date: Fri, 17 Jan 2020 15:14:53 +0200 Subject: [PATCH 225/274] Fixed bug CORE-6231 : access violation on shutdown of xnet connection to local database when events have been registered --- src/remote/client/interface.cpp | 9 ++++++++- src/remote/os/win32/wnet.cpp | 15 ++++++++++++--- src/remote/os/win32/xnet.cpp | 11 +++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/remote/client/interface.cpp b/src/remote/client/interface.cpp index 18fa388fbb..abb2ecbfda 100644 --- a/src/remote/client/interface.cpp +++ b/src/remote/client/interface.cpp @@ -7204,7 +7204,14 @@ static THREAD_ENTRY_DECLARE event_thread(THREAD_ENTRY_PARAM arg) P_OP operation = op_void; { // scope RefMutexGuard portGuard(*port->port_sync, FB_FUNCTION); - stuff = port->receive(&packet); + try + { + stuff = port->receive(&packet); + } + catch(status_exception&) + { + // ignore + } operation = packet.p_operation; diff --git a/src/remote/os/win32/wnet.cpp b/src/remote/os/win32/wnet.cpp index b551112bd9..11c1fb939c 100644 --- a/src/remote/os/win32/wnet.cpp +++ b/src/remote/os/win32/wnet.cpp @@ -759,7 +759,11 @@ static void disconnect(rem_port* port) } wnet_ports->unRegisterPort(port); - port->release(); + + if (port->port_thread_guard && port->port_events_thread && !Thread::isCurrent(port->port_events_threadId)) + port->port_thread_guard->setWait(port->port_events_thread); + else + port->release(); } @@ -1332,7 +1336,7 @@ static bool packet_receive(rem_port* port, UCHAR* buffer, SSHORT buffer_length, if (!n) { - if (port->port_flags & PORT_detached) + if (port->port_flags & (PORT_detached | PORT_disconnect)) return false; return wnet_error(port, "ReadFile end-of-file", isc_net_read_err, dwError); @@ -1408,10 +1412,15 @@ static bool packet_send( rem_port* port, const SCHAR* buffer, SSHORT buffer_leng status = GetOverlappedResult(port->port_pipe, &ovrl, &n, TRUE); dwError = GetLastError(); } - if (!status) + if (!status && dwError != ERROR_NO_DATA) return wnet_error(port, "WriteFile", isc_net_write_err, dwError); if (n != length) + { + if (port->port_flags & (PORT_detached | PORT_disconnect)) + return false; + return wnet_error(port, "WriteFile truncated", isc_net_write_err, dwError); + } #if defined(DEBUG) && defined(WNET_trace) packet_print("send", reinterpret_cast(buffer), buffer_length); diff --git a/src/remote/os/win32/xnet.cpp b/src/remote/os/win32/xnet.cpp index 7385d0c09e..482f14a1f4 100644 --- a/src/remote/os/win32/xnet.cpp +++ b/src/remote/os/win32/xnet.cpp @@ -1068,6 +1068,14 @@ static void cleanup_port(rem_port* port) * **************************************/ + if (port->port_thread_guard && port->port_events_thread && !Thread::isCurrent(port->port_events_threadId)) + { + //port->port_thread_guard->setWait(port->port_events_thread); + + // Do not release XNET structures while event's thread working + Thread::waitForCompletion(port->port_events_thread); + } + if (port->port_xcc) { cleanup_comm(port->port_xcc); @@ -1996,6 +2004,9 @@ static bool_t xnet_read(XDR* xdrs) const DWORD wait_result = WaitForSingleObject(xcc->xcc_event_recv_channel_filled, XNET_RECV_WAIT_TIMEOUT); + if (port->port_flags & PORT_disconnect) + return FALSE; + if (wait_result == WAIT_OBJECT_0) { // Client has written some data for us (server) to read From 1f32a12b90c4d41307e83d6c6003d36b99133749 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 18 Jan 2020 00:04:35 +0000 Subject: [PATCH 226/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 840d7d5a09..0b3ba635c8 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1731 + FORMAL BUILD NUMBER:1732 */ -#define PRODUCT_VER_STRING "4.0.0.1731" -#define FILE_VER_STRING "WI-T4.0.0.1731" -#define LICENSE_VER_STRING "WI-T4.0.0.1731" -#define FILE_VER_NUMBER 4, 0, 0, 1731 +#define PRODUCT_VER_STRING "4.0.0.1732" +#define FILE_VER_STRING "WI-T4.0.0.1732" +#define LICENSE_VER_STRING "WI-T4.0.0.1732" +#define FILE_VER_NUMBER 4, 0, 0, 1732 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1731" +#define FB_BUILD_NO "1732" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 263b7217af..9fe5324fae 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1731 +BuildNum=1732 NowAt=`pwd` cd `dirname $0` From 2a6d7097448aee29435369f1fb8c1429bf04b287 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Sun, 19 Jan 2020 12:35:28 +0300 Subject: [PATCH 227/274] Fixed wrong parameters order. Thanks to DS --- src/burp/restore.epp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/burp/restore.epp b/src/burp/restore.epp index 9493ae7fa8..0cce0e69f0 100644 --- a/src/burp/restore.epp +++ b/src/burp/restore.epp @@ -1090,7 +1090,7 @@ void create_database(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TE if (!tdgbl->gbl_sw_crypt) { - BURP_error(true, 378); + BURP_error(378, true); // Unknown crypt plugin name - use -CRYPT switch } From a238ca5eba6d259565138bafe3098d69bcf47d6d Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Mon, 20 Jan 2020 00:04:31 +0000 Subject: [PATCH 228/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 0b3ba635c8..067c443900 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1732 + FORMAL BUILD NUMBER:1733 */ -#define PRODUCT_VER_STRING "4.0.0.1732" -#define FILE_VER_STRING "WI-T4.0.0.1732" -#define LICENSE_VER_STRING "WI-T4.0.0.1732" -#define FILE_VER_NUMBER 4, 0, 0, 1732 +#define PRODUCT_VER_STRING "4.0.0.1733" +#define FILE_VER_STRING "WI-T4.0.0.1733" +#define LICENSE_VER_STRING "WI-T4.0.0.1733" +#define FILE_VER_NUMBER 4, 0, 0, 1733 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1732" +#define FB_BUILD_NO "1733" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 9fe5324fae..940745e82a 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1732 +BuildNum=1733 NowAt=`pwd` cd `dirname $0` From 5f1ea284dc27e5421405d7c8148550e0a98aa47d Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sun, 19 Jan 2020 23:08:32 -0300 Subject: [PATCH 229/274] Fixed CORE-6116 - The Metadata script extracted using ISQL of a database restored from a Firebird 2.5.9 Backup is invalid/incorrect when table has COMPUTED BY field. --- src/isql/extract.epp | 44 +++++++++++++++++++++++++++---------------- src/isql/isql.epp | 28 +++++++++++++++++++++++++++ src/isql/isql_proto.h | 4 ++++ 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/src/isql/extract.epp b/src/isql/extract.epp index 33ecd7a499..1b720d6822 100644 --- a/src/isql/extract.epp +++ b/src/isql/extract.epp @@ -417,10 +417,13 @@ int EXTRACT_list_table(const SCHAR* relation_name, if ((FLD.RDB$FIELD_TYPE == T_CHAR) || (FLD.RDB$FIELD_TYPE == VARCHAR)) { - if (FLD.RDB$CHARACTER_LENGTH.NULL) - isqlGlob.printf("(%d)", FLD.RDB$FIELD_LENGTH); - else - isqlGlob.printf("(%d)", FLD.RDB$CHARACTER_LENGTH); + isqlGlob.printf("(%d)", + ISQL_get_char_length( + FLD.RDB$FIELD_LENGTH, + FLD.RDB$CHARACTER_LENGTH.NULL, FLD.RDB$CHARACTER_LENGTH, + FLD.RDB$CHARACTER_SET_ID.NULL, FLD.RDB$CHARACTER_SET_ID + ) + ); } // Catch arrays after printing the type @@ -833,10 +836,13 @@ static void get_procedure_args(const char* proc_name) // FSG 18.Nov.2000 if ((FLD.RDB$FIELD_TYPE == T_CHAR) || (FLD.RDB$FIELD_TYPE == VARCHAR)) { - if (FLD.RDB$CHARACTER_LENGTH.NULL) - isqlGlob.printf("(%d)", FLD.RDB$FIELD_LENGTH); - else - isqlGlob.printf("(%d)", FLD.RDB$CHARACTER_LENGTH); + isqlGlob.printf("(%d)", + ISQL_get_char_length( + FLD.RDB$FIELD_LENGTH, + FLD.RDB$CHARACTER_LENGTH.NULL, FLD.RDB$CHARACTER_LENGTH, + FLD.RDB$CHARACTER_SET_ID.NULL, FLD.RDB$CHARACTER_SET_ID + ) + ); } // Show international character sets and collations @@ -1054,10 +1060,13 @@ static void get_function_args_ods12(const char* func_name, USHORT out_arg) // FSG 18.Nov.2000 if ((FLD.RDB$FIELD_TYPE == T_CHAR) || (FLD.RDB$FIELD_TYPE == VARCHAR)) { - if (FLD.RDB$CHARACTER_LENGTH.NULL) - isqlGlob.printf("(%d)", FLD.RDB$FIELD_LENGTH); - else - isqlGlob.printf("(%d)", FLD.RDB$CHARACTER_LENGTH); + isqlGlob.printf("(%d)", + ISQL_get_char_length( + FLD.RDB$FIELD_LENGTH, + FLD.RDB$CHARACTER_LENGTH.NULL, FLD.RDB$CHARACTER_LENGTH, + FLD.RDB$CHARACTER_SET_ID.NULL, FLD.RDB$CHARACTER_SET_ID + ) + ); } // Show international character sets and collations @@ -2624,10 +2633,13 @@ static void listRelationComputed(LegacyTables flag, SSHORT default_char_set_id) if ((FLD.RDB$FIELD_TYPE == T_CHAR) || (FLD.RDB$FIELD_TYPE == VARCHAR)) { - if (FLD.RDB$CHARACTER_LENGTH.NULL) - isqlGlob.printf("(%d)", FLD.RDB$FIELD_LENGTH); - else - isqlGlob.printf("(%d)", FLD.RDB$CHARACTER_LENGTH); + isqlGlob.printf("(%d)", + ISQL_get_char_length( + FLD.RDB$FIELD_LENGTH, + FLD.RDB$CHARACTER_LENGTH.NULL, FLD.RDB$CHARACTER_LENGTH, + FLD.RDB$CHARACTER_SET_ID.NULL, FLD.RDB$CHARACTER_SET_ID + ) + ); } // Catch arrays after printing the type diff --git a/src/isql/isql.epp b/src/isql/isql.epp index 033194a87f..059d4a8d9e 100644 --- a/src/isql/isql.epp +++ b/src/isql/isql.epp @@ -1211,6 +1211,34 @@ SSHORT ISQL_get_field_length(const TEXT* field_name) } +SSHORT ISQL_get_char_length( + SSHORT fieldLength, + SSHORT characterLengthNull, SSHORT characterLength, + SSHORT characterSetIdNull, SSHORT characterSetId) +{ + if (characterLengthNull || characterLength == 0) + { + if (!characterSetIdNull) + { + FOR CS IN RDB$CHARACTER_SETS + WITH CS.RDB$CHARACTER_SET_ID EQ characterSetId AND + CS.RDB$BYTES_PER_CHARACTER > 0 + { + fieldLength /= CS.RDB$BYTES_PER_CHARACTER; + } + END_FOR + ON_ERROR + ISQL_errmsg(fbStatus); + END_ERROR; + } + + return fieldLength; + } + else + return characterLength; +} + + void ISQL_get_character_sets(SSHORT char_set_id, SSHORT collation, bool collate_only, bool not_null, bool quote, TEXT* string) { diff --git a/src/isql/isql_proto.h b/src/isql/isql_proto.h index 5a7b79c536..52a83c5cfa 100644 --- a/src/isql/isql_proto.h +++ b/src/isql/isql_proto.h @@ -45,6 +45,10 @@ void ISQL_get_character_sets(SSHORT, SSHORT, bool, bool, bool, TEXT*); SSHORT ISQL_get_default_char_set_id(); void ISQL_get_default_source(const TEXT*, TEXT*, ISC_QUAD*); SSHORT ISQL_get_field_length(const TEXT*); +SSHORT ISQL_get_char_length( + SSHORT fieldLength, + SSHORT characterLengthNull, SSHORT characterLength, + SSHORT characterSetIdNull, SSHORT characterSetId); SLONG ISQL_get_index_segments(TEXT*, const size_t, const TEXT*, bool); bool ISQL_get_null_flag(const TEXT*, TEXT*); void ISQL_get_version(bool); From d24b4c96bd0a0a2dadcf1018b0686b8fe0799722 Mon Sep 17 00:00:00 2001 From: hvlad Date: Mon, 20 Jan 2020 14:31:42 +0200 Subject: [PATCH 230/274] Call of TimeZoneUtil::initTimeZoneEnv() is moved to the InitPrefix::init() to avoid too early initialization of default config file on Windows. Current code attempts to load firebird.conf from .exe folder (should be loaded from fbclient.dll folder). --- src/yvalve/gds.cpp | 2 ++ src/yvalve/why.cpp | 14 -------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/yvalve/gds.cpp b/src/yvalve/gds.cpp index c0dd855594..afab931314 100644 --- a/src/yvalve/gds.cpp +++ b/src/yvalve/gds.cpp @@ -3930,6 +3930,8 @@ public: } msgPrefix.copyTo(fb_prefix_msg_val, sizeof(fb_prefix_msg_val)); fb_prefix_msg = fb_prefix_msg_val; + + TimeZoneUtil::initTimeZoneEnv(); } static void cleanup() { diff --git a/src/yvalve/why.cpp b/src/yvalve/why.cpp index 1fc9754034..1df43ca04b 100644 --- a/src/yvalve/why.cpp +++ b/src/yvalve/why.cpp @@ -40,7 +40,6 @@ #include "../common/StatementMetadata.h" #include "../common/StatusHolder.h" #include "../common/ThreadStart.h" -#include "../common/TimeZoneUtil.h" #include "../common/isc_proto.h" #include "../common/isc_f_proto.h" #include "../common/utils_proto.h" @@ -738,19 +737,6 @@ RefPtr translateHandle(GlobalPtr //------------------------------------- -class TimeZoneDataInit -{ -public: - explicit TimeZoneDataInit(MemoryPool&) - { - TimeZoneUtil::initTimeZoneEnv(); - } -}; - -static GlobalPtr timeZoneDataInit; - -//------------------------------------- - const int SHUTDOWN_TIMEOUT = 5000; // 5 sec class ShutdownInit From d8be3d534fcafc2e8898013d6da566a88b33da97 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Mon, 20 Jan 2020 19:19:48 +0300 Subject: [PATCH 231/274] Add chacha to default config and use it when available. Clients, missing required plugin, will continue using rc4. --- builds/install/misc/firebird.conf | 12 +++++++----- src/common/config/config.cpp | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/builds/install/misc/firebird.conf b/builds/install/misc/firebird.conf index f627cd464e..897e583e66 100644 --- a/builds/install/misc/firebird.conf +++ b/builds/install/misc/firebird.conf @@ -446,8 +446,8 @@ #AuthClient = Srp256, Srp, Legacy_Auth #Non Windows clients #AuthClient = Srp256, Srp, Win_Sspi, Legacy_Auth #Windows clients # -# If you need to use server plugins that do not provide encryption key (both Legacy_Auth -# & Win_Sspi) you should also turn off required encryption on the wire with WireCrypt +# If you need to use server plugins that do not provide encryption key (Legacy_Auth is the +# only such std plugin) you should also turn off required encryption on the wire with WireCrypt # configuration parameter except when working with the XNET protocol which is never encrypted. # @@ -466,12 +466,14 @@ #TracePlugin = fbtrace # Wire crypt plugins are used to crypt data transferred over the wire. -# In default case wire is encrypted using Alleged RC4 -# (key must be generated by auth plugin). +# In default case wire is encrypted using ChaCha#20 or Alleged RC4. +# Key must be generated by auth plugin. +# For chacha we are using 16 or 32 bytes key (depends upon what is provided +# by auth plugin), 12 bytes nonce and 4 bytes counter, 20 (10 + 10) rounds are made. # # Per-connection configurable. # -#WireCryptPlugin = Arc4 +#WireCryptPlugin = ChaCha, Arc4 # Key holder is a kind of temp storage for DB crypt keys. # There is no default for this kind of plugins. diff --git a/src/common/config/config.cpp b/src/common/config/config.cpp index ddfe804adb..e2573d8140 100644 --- a/src/common/config/config.cpp +++ b/src/common/config/config.cpp @@ -204,7 +204,7 @@ const Config::ConfigEntry Config::entries[MAX_CONFIG_KEY] = {TYPE_STRING, "SecurityDatabase", (ConfigValue) "security.db"}, // sec/db alias - rely on databases.conf {TYPE_STRING, "ServerMode", (ConfigValue) ""}, // actual value differs in boot/regular cases {TYPE_STRING, "WireCrypt", (ConfigValue) NULL}, - {TYPE_STRING, "WireCryptPlugin", (ConfigValue) "Arc4"}, + {TYPE_STRING, "WireCryptPlugin", (ConfigValue) "ChaCha, Arc4"}, {TYPE_STRING, "KeyHolderPlugin", (ConfigValue) ""}, {TYPE_BOOLEAN, "RemoteAccess", (ConfigValue) true}, {TYPE_BOOLEAN, "IPv6V6Only", (ConfigValue) false}, From 0a48a5e8357cf21a25edd83b70a5d8b158d76fb8 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Mon, 20 Jan 2020 20:18:15 +0300 Subject: [PATCH 232/274] This should fix MacOS build --- src/plugins/crypt/chacha/ChaCha.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/crypt/chacha/ChaCha.cpp b/src/plugins/crypt/chacha/ChaCha.cpp index 7331c6a899..218bd3c7c7 100644 --- a/src/plugins/crypt/chacha/ChaCha.cpp +++ b/src/plugins/crypt/chacha/ChaCha.cpp @@ -168,7 +168,7 @@ SimpleFactory factory; } // anonymous namespace -extern "C" void FB_PLUGIN_ENTRY_POINT(Firebird::IMaster* master) +extern "C" void FB_EXPORTED FB_PLUGIN_ENTRY_POINT(Firebird::IMaster* master) { CachedMasterInterface::set(master); PluginManagerInterfacePtr()->registerPluginFactory(IPluginManager::TYPE_WIRE_CRYPT, "ChaCha", &factory); From a31b3410cc3fc84b23593f6d28205e404cfd0782 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Tue, 21 Jan 2020 00:04:40 +0000 Subject: [PATCH 233/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 067c443900..35093fe8b7 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1733 + FORMAL BUILD NUMBER:1737 */ -#define PRODUCT_VER_STRING "4.0.0.1733" -#define FILE_VER_STRING "WI-T4.0.0.1733" -#define LICENSE_VER_STRING "WI-T4.0.0.1733" -#define FILE_VER_NUMBER 4, 0, 0, 1733 +#define PRODUCT_VER_STRING "4.0.0.1737" +#define FILE_VER_STRING "WI-T4.0.0.1737" +#define LICENSE_VER_STRING "WI-T4.0.0.1737" +#define FILE_VER_NUMBER 4, 0, 0, 1737 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1733" +#define FB_BUILD_NO "1737" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 940745e82a..47d4f35077 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1733 +BuildNum=1737 NowAt=`pwd` cd `dirname $0` From 5d1bcc284c6e444b6232dd544c4d81dd6c922807 Mon Sep 17 00:00:00 2001 From: Dimitry Sibiryakov Date: Mon, 20 Jan 2020 17:36:24 +0100 Subject: [PATCH 234/274] Generate ids.h without m4 --- builds/mac_os_x/CS/CS.pbproj/project.pbxproj | 5 - builds/posix/Makefile.in | 17 +- src/include/gen/ids.h | 708 ------------------- src/jrd/DbCreators.cpp | 2 +- src/jrd/Mapping.cpp | 2 +- src/jrd/TimeZone.cpp | 2 +- src/jrd/UserManagement.cpp | 2 +- src/jrd/ids.h | 89 +++ src/jrd/ini.epp | 2 +- src/jrd/replication/Applier.cpp | 2 +- src/jrd/vio.cpp | 2 +- src/misc/ids.m | 30 - 12 files changed, 97 insertions(+), 766 deletions(-) delete mode 100644 src/include/gen/ids.h create mode 100644 src/jrd/ids.h delete mode 100644 src/misc/ids.m diff --git a/builds/mac_os_x/CS/CS.pbproj/project.pbxproj b/builds/mac_os_x/CS/CS.pbproj/project.pbxproj index 9c84eabf94..82eb6a0601 100644 --- a/builds/mac_os_x/CS/CS.pbproj/project.pbxproj +++ b/builds/mac_os_x/CS/CS.pbproj/project.pbxproj @@ -4411,11 +4411,6 @@ path = ids.h; refType = 4; }; - F616C6050200B0CF01EF0ADE = { - isa = PBXFileReference; - path = ids.m; - refType = 4; - }; F616C6060200B0CF01EF0ADE = { isa = PBXFileReference; path = idx.cpp; diff --git a/builds/posix/Makefile.in b/builds/posix/Makefile.in index 1d97d66a6b..3f05cd22ac 100644 --- a/builds/posix/Makefile.in +++ b/builds/posix/Makefile.in @@ -226,7 +226,7 @@ $(TOMCRYPT_LIB): $(TOM_Objs) # main build target for both debug and release builds # -.PHONY: cross1 cross2 boot yvalve engine fbintl gpre utilities plugins rest codes ids examples cross_rest +.PHONY: cross1 cross2 boot yvalve engine fbintl gpre utilities plugins rest codes examples cross_rest master_process: ln -sf $(SRC_ROOT)/include/gen/autoconfig.auto $(SRC_ROOT)/include/gen/autoconfig.h @@ -236,10 +236,6 @@ master_process: $(MAKE) updateCloopInterfaces $(MAKE) boot $(MAKE) yvalve -ifeq ($(IsDeveloper), Y) -# In developer mode we must regenerate various files in include/gen - $(MAKE) ids -endif $(MAKE) engine $(MAKE) fbintl $(MAKE) utilities @@ -271,10 +267,6 @@ cross1: $(MAKE) updateCloopInterfaces $(MAKE) boot $(MAKE) yvalve -ifeq ($(IsDeveloper), Y) -# In developer mode we must regenerate various files in include/gen - $(MAKE) ids -endif $(MAKE) engine $(MAKE) fbintl $(MAKE) gbak isql gfix @@ -590,22 +582,15 @@ $(COMPAT_SQL): $(SRC_COMPAT_SQL) .PHONY: gen_codes CODES = $(BIN)/codes$(EXEC_EXT) -IDS = $(SRC_ROOT)/include/gen/ids.h codes: gen_codes -ids: $(IDS) - gen_codes: $(CODES) msg.timestamp $(CODES) $(SRC_ROOT)/include/gen $(LNG_ROOT) $(CODES): $(CODES_Objects) $(COMMON_LIB) $(EXE_LINK) $(EXE_LINK_OPTIONS) $^ -o $@ $(FIREBIRD_LIBRARY_LINK) $(LINK_LIBS) $(call LINK_DARWIN_RPATH,..) -$(IDS): $(SRC_ROOT)/misc/ids.m $(SRC_ROOT)/jrd/relations.h - m4 $< > $@ - - #___________________________________________________________________________ # all the rest we need to build # diff --git a/src/include/gen/ids.h b/src/include/gen/ids.h deleted file mode 100644 index 31cf55039c..0000000000 --- a/src/include/gen/ids.h +++ /dev/null @@ -1,708 +0,0 @@ - -/* - * PROGRAM: JRD Access Method - * MODULE: relations.h - * DESCRIPTION: System relation definitions - * - * The contents of this file are subject to the Interbase Public - * License Version 1.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy - * of the License at http://www.Inprise.com/IPL.html - * - * Software distributed under the License is distributed on an - * "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express - * or implied. See the License for the specific language governing - * rights and limitations under the License. - * - * The Original Code was created by Inprise Corporation - * and its predecessors. Portions created by Inprise Corporation are - * Copyright (C) Inprise Corporation. - * - * All Rights Reserved. - * Contributor(s): ______________________________________. - */ - -// Order of relations in this file affect their IDs, -// so please add new relations to the end of the list - -// Relation 0 (RDB$PAGES) - - const USHORT f_pag_page = 0; - const USHORT f_pag_id = 1; - const USHORT f_pag_seq = 2; - const USHORT f_pag_type = 3; - - -// Relation 1 (RDB$DATABASE) - - const USHORT f_dat_desc = 0; - const USHORT f_dat_id = 1; - const USHORT f_dat_class = 2; - const USHORT f_dat_charset = 3; - const USHORT f_dat_linger = 4; - const USHORT f_dat_sql_security = 5; - - -// Relation 2 (RDB$FIELDS) - - const USHORT f_fld_name = 0; - const USHORT f_fld_qname = 1; - const USHORT f_fld_v_blr = 2; - const USHORT f_fld_v_source = 3; - const USHORT f_fld_computed = 4; - const USHORT f_fld_csource = 5; - const USHORT f_fld_default = 6; - const USHORT f_fld_dsource = 7; - const USHORT f_fld_length = 8; - const USHORT f_fld_scale = 9; - const USHORT f_fld_type = 10; - const USHORT f_fld_sub_type = 11; - const USHORT f_fld_missing = 12; - const USHORT f_fld_msource = 13; - const USHORT f_fld_desc = 14; - const USHORT f_fld_sys_flag = 15; - const USHORT f_fld_qheader = 16; - const USHORT f_fld_seg_len = 17; - const USHORT f_fld_estring = 18; - const USHORT f_fld_ext_length = 19; - const USHORT f_fld_ext_scale = 20; - const USHORT f_fld_ext_type = 21; - const USHORT f_fld_dimensions = 22; - const USHORT f_fld_null_flag = 23; - const USHORT f_fld_char_length = 24; - const USHORT f_fld_coll_id = 25; - const USHORT f_fld_charset_id = 26; - const USHORT f_fld_precision = 27; - const USHORT f_fld_class = 28; - const USHORT f_fld_owner = 29; - - -// Relation 3 (RDB$INDEX_SEGMENTS) - - const USHORT f_seg_name = 0; - const USHORT f_seg_field = 1; - const USHORT f_seg_position = 2; - const USHORT f_seg_statistics = 3; - - -// Relation 4 (RDB$INDICES) - - const USHORT f_idx_name = 0; - const USHORT f_idx_relation = 1; - const USHORT f_idx_id = 2; - const USHORT f_idx_flag = 3; - const USHORT f_idx_desc = 4; - const USHORT f_idx_count = 5; - const USHORT f_idx_inactive = 6; - const USHORT f_idx_type = 7; - const USHORT f_idx_foreign = 8; - const USHORT f_idx_sys_flag = 9; - const USHORT f_idx_exp_blr = 10; - const USHORT f_idx_exp_source = 11; - const USHORT f_idx_statistics = 12; - - -// Relation 5 (RDB$RELATION_FIELDS) - - const USHORT f_rfr_fname = 0; - const USHORT f_rfr_rname = 1; - const USHORT f_rfr_sname = 2; - const USHORT f_rfr_qname = 3; - const USHORT f_rfr_base = 4; - const USHORT f_rfr_estring = 5; - const USHORT f_rfr_position = 6; - const USHORT f_rfr_qheader = 7; - const USHORT f_rfr_flag = 8; - const USHORT f_rfr_id = 9; - const USHORT f_rfr_context = 10; - const USHORT f_rfr_desc = 11; - const USHORT f_rfr_default = 12; - const USHORT f_rfr_sys_flag = 13; - const USHORT f_rfr_class = 14; - const USHORT f_rfr_complex = 15; - const USHORT f_rfr_null_flag = 16; - const USHORT f_rfr_dsource = 17; - const USHORT f_rfr_coll_id = 18; - const USHORT f_rfr_gen_name = 19; - const USHORT f_rfr_identity_type = 20; - - -// Relation 6 (RDB$RELATIONS) - - const USHORT f_rel_blr = 0; - const USHORT f_rel_source = 1; - const USHORT f_rel_desc = 2; - const USHORT f_rel_id = 3; - const USHORT f_rel_sys_flag = 4; - const USHORT f_rel_key_len = 5; - const USHORT f_rel_format = 6; - const USHORT f_rel_field_id = 7; - const USHORT f_rel_name = 8; - const USHORT f_rel_class = 9; - const USHORT f_rel_ext_file = 10; - const USHORT f_rel_runtime = 11; - const USHORT f_rel_ext_desc = 12; - const USHORT f_rel_owner = 13; - const USHORT f_rel_def_class = 14; - const USHORT f_rel_flags = 15; - const USHORT f_rel_type = 16; - const USHORT f_rel_sql_security = 17; - - -// Relation 7 (RDB$VIEW_RELATIONS) - - const USHORT f_vrl_vname = 0; - const USHORT f_vrl_rname = 1; - const USHORT f_vrl_context = 2; - const USHORT f_vrl_cname = 3; - const USHORT f_vrl_context_type = 4; - const USHORT f_vrl_pkg_name = 5; - - -// Relation 8 (RDB$FORMATS) - - const USHORT f_fmt_rid = 0; - const USHORT f_fmt_format = 1; - const USHORT f_fmt_desc = 2; - - -// Relation 9 (RDB$SECURITY_CLASSES) - - const USHORT f_cls_class = 0; - const USHORT f_cls_acl = 1; - const USHORT f_cls_desc = 2; - - -// Relation 10 (RDB$FILES) - - const USHORT f_file_name = 0; - const USHORT f_file_seq = 1; - const USHORT f_file_start = 2; - const USHORT f_file_length = 3; - const USHORT f_file_flags = 4; - const USHORT f_file_shad_num = 5; - - -// Relation 11 (RDB$TYPES) - - const USHORT f_typ_field = 0; - const USHORT f_typ_type = 1; - const USHORT f_typ_name = 2; - const USHORT f_typ_desc = 3; - const USHORT f_typ_sys_flag = 4; - - -// Relation 12 (RDB$TRIGGERS) - - const USHORT f_trg_name = 0; - const USHORT f_trg_rname = 1; - const USHORT f_trg_seq = 2; - const USHORT f_trg_type = 3; - const USHORT f_trg_source = 4; - const USHORT f_trg_blr = 5; - const USHORT f_trg_desc = 6; - const USHORT f_trg_inactive = 7; - const USHORT f_trg_sys_flag = 8; - const USHORT f_trg_flags = 9; - const USHORT f_trg_valid_blr = 10; - const USHORT f_trg_debug_info = 11; - const USHORT f_trg_engine_name = 12; - const USHORT f_trg_entry = 13; - const USHORT f_trg_sql_security = 14; - - -// Relation 13 (RDB$DEPENDENCIES) - - const USHORT f_dpd_name = 0; - const USHORT f_dpd_o_name = 1; - const USHORT f_dpd_f_name = 2; - const USHORT f_dpd_type = 3; - const USHORT f_dpd_o_type = 4; - const USHORT f_dpd_pkg_name = 5; - - -// Relation 14 (RDB$FUNCTIONS) - - const USHORT f_fun_name = 0; - const USHORT f_fun_type = 1; - const USHORT f_fun_qname = 2; - const USHORT f_fun_desc = 3; - const USHORT f_fun_module = 4; - const USHORT f_fun_entry = 5; - const USHORT f_fun_ret_arg = 6; - const USHORT f_fun_sys_flag = 7; - const USHORT f_fun_engine_name = 8; - const USHORT f_fun_pkg_name = 9; - const USHORT f_fun_private_flag = 10; - const USHORT f_fun_source = 11; - const USHORT f_fun_id = 12; - const USHORT f_fun_blr = 13; - const USHORT f_fun_valid_blr = 14; - const USHORT f_fun_debug_info = 15; - const USHORT f_fun_class = 16; - const USHORT f_fun_owner = 17; - const USHORT f_fun_legacy_flag = 18; - const USHORT f_fun_deterministic_flag = 19; - const USHORT f_fun_sql_security = 20; - - -// Relation 15 (RDB$FUNCTION_ARGUMENTS) - - const USHORT f_arg_fun_name = 0; - const USHORT f_arg_pos = 1; - const USHORT f_arg_mech = 2; - const USHORT f_arg_type = 3; - const USHORT f_arg_scale = 4; - const USHORT f_arg_length = 5; - const USHORT f_arg_sub_type = 6; - const USHORT f_arg_charset_id = 7; - const USHORT f_arg_precision = 8; - const USHORT f_arg_char_length = 9; - const USHORT f_arg_pkg_name = 10; - const USHORT f_arg_name = 11; - const USHORT f_arg_sname = 12; - const USHORT f_arg_default = 13; - const USHORT f_arg_dsource = 14; - const USHORT f_arg_coll_id = 15; - const USHORT f_arg_null_flag = 16; - const USHORT f_arg_arg_mech = 17; - const USHORT f_arg_fname = 18; - const USHORT f_arg_rname = 19; - const USHORT f_arg_sys_flag = 20; - const USHORT f_arg_desc = 21; - - -// Relation 16 (RDB$FILTERS) - - const USHORT f_flt_name = 0; - const USHORT f_flt_desc = 1; - const USHORT f_flt_module = 2; - const USHORT f_flt_entry = 3; - const USHORT f_flt_input = 4; - const USHORT f_flt_output = 5; - const USHORT f_flt_sys_flag = 6; - const USHORT f_flt_class = 7; - const USHORT f_flt_owner = 8; - - -// Relation 17 (RDB$TRIGGER_MESSAGES) - - const USHORT f_msg_trigger = 0; - const USHORT f_msg_number = 1; - const USHORT f_msg_msg = 2; - - -// Relation 18 (RDB$USER_PRIVILEGES) - - const USHORT f_prv_user = 0; - const USHORT f_prv_grantor = 1; - const USHORT f_prv_priv = 2; - const USHORT f_prv_grant = 3; - const USHORT f_prv_rname = 4; - const USHORT f_prv_fname = 5; - const USHORT f_prv_u_type = 6; - const USHORT f_prv_o_type = 7; - - -// Relation 19 (RDB$TRANSACTIONS) - - const USHORT f_trn_id = 0; - const USHORT f_trn_state = 1; - const USHORT f_trn_time = 2; - const USHORT f_trn_desc = 3; - - -// Relation 20 (RDB$GENERATORS) - - const USHORT f_gen_name = 0; - const USHORT f_gen_id = 1; - const USHORT f_gen_sys_flag = 2; - const USHORT f_gen_desc = 3; - const USHORT f_gen_class = 4; - const USHORT f_gen_owner = 5; - const USHORT f_gen_init_val = 6; - const USHORT f_gen_increment = 7; - - -// Relation 21 (RDB$FIELD_DIMENSIONS) - - const USHORT f_dims_fname = 0; - const USHORT f_dims_dim = 1; - const USHORT f_dims_lower = 2; - const USHORT f_dims_upper = 3; - - -// Relation 22 (RDB$RELATION_CONSTRAINTS) - - const USHORT f_rcon_cname = 0; - const USHORT f_rcon_ctype = 1; - const USHORT f_rcon_rname = 2; - const USHORT f_rcon_dfr = 3; - const USHORT f_rcon_idfr = 4; - const USHORT f_rcon_iname = 5; - - -// Relation 23 (RDB$REF_CONSTRAINTS) - - const USHORT f_refc_cname = 0; - const USHORT f_refc_uq = 1; - const USHORT f_refc_match = 2; - const USHORT f_refc_upd_rul = 3; - const USHORT f_refc_del_rul = 4; - - -// Relation 24 (RDB$CHECK_CONSTRAINTS) - - const USHORT f_ccon_cname = 0; - const USHORT f_ccon_tname = 1; - - -// Relation 25 (RDB$LOG_FILES) - - const USHORT f_log_name = 0; - const USHORT f_log_seq = 1; - const USHORT f_log_length = 2; - const USHORT f_log_partitions = 3; - const USHORT f_log_p_offset = 4; - const USHORT f_log_flags = 5; - - -// Relation 26 (RDB$PROCEDURES) - - const USHORT f_prc_name = 0; - const USHORT f_prc_id = 1; - const USHORT f_prc_inputs = 2; - const USHORT f_prc_outputs = 3; - const USHORT f_prc_desc = 4; - const USHORT f_prc_source = 5; - const USHORT f_prc_blr = 6; - const USHORT f_prc_class = 7; - const USHORT f_prc_owner = 8; - const USHORT f_prc_runtime = 9; - const USHORT f_prc_sys_flag = 10; - const USHORT f_prc_type = 11; - const USHORT f_prc_valid_blr = 12; - const USHORT f_prc_debug_info = 13; - const USHORT f_prc_engine_name = 14; - const USHORT f_prc_entry = 15; - const USHORT f_prc_pkg_name = 16; - const USHORT f_prc_private_flag = 17; - const USHORT f_prc_sql_security = 18; - - -// Relation 27 (RDB$PROCEDURE_PARAMETERS) - - const USHORT f_prm_name = 0; - const USHORT f_prm_procedure = 1; - const USHORT f_prm_number = 2; - const USHORT f_prm_type = 3; - const USHORT f_prm_sname = 4; - const USHORT f_prm_desc = 5; - const USHORT f_prm_sys_flag = 6; - const USHORT f_prm_default = 7; - const USHORT f_prm_dsource = 8; - const USHORT f_prm_coll_id = 9; - const USHORT f_prm_null_flag = 10; - const USHORT f_prm_mech = 11; - const USHORT f_prm_fname = 12; - const USHORT f_prm_rname = 13; - const USHORT f_prm_pkg_name = 14; - - -// Relation 28 (RDB$CHARACTER_SETS) - - const USHORT f_cs_cs_name = 0; - const USHORT f_cs_form_of_use = 1; - const USHORT f_cs_num_chars = 2; - const USHORT f_cs_def_collate = 3; - const USHORT f_cs_id = 4; - const USHORT f_cs_sys_flag = 5; - const USHORT f_cs_desc = 6; - const USHORT f_cs_fun_name = 7; - const USHORT f_cs_bytes_char = 8; - const USHORT f_cs_class = 9; - const USHORT f_cs_owner = 10; - - -// Relation 29 (RDB$COLLATIONS) - - const USHORT f_coll_name = 0; - const USHORT f_coll_id = 1; - const USHORT f_coll_cs_id = 2; - const USHORT f_coll_attr = 3; - const USHORT f_coll_sys_flag = 4; - const USHORT f_coll_desc = 5; - const USHORT f_coll_fun_name = 6; - const USHORT f_coll_base_collation_name = 7; - const USHORT f_coll_specific_attr = 8; - const USHORT f_coll_class = 9; - const USHORT f_coll_owner = 10; - - -// Relation 30 (RDB$EXCEPTIONS) - - const USHORT f_xcp_name = 0; - const USHORT f_xcp_number = 1; - const USHORT f_xcp_msg = 2; - const USHORT f_xcp_desc = 3; - const USHORT f_xcp_sys_flag = 4; - const USHORT f_xcp_class = 5; - const USHORT f_xcp_owner = 6; - - -// Relation 31 (RDB$ROLES) - - const USHORT f_rol_name = 0; - const USHORT f_rol_owner = 1; - const USHORT f_rol_desc = 2; - const USHORT f_rol_sys_flag = 3; - const USHORT f_rol_class = 4; - const USHORT f_rol_sys_priv = 5; - - -// Relation 32 (RDB$BACKUP_HISTORY) - - const USHORT f_backup_id = 0; - const USHORT f_backup_time = 1; - const USHORT f_backup_level = 2; - const USHORT f_backup_guid = 3; - const USHORT f_backup_scn = 4; - const USHORT f_backup_name = 5; - - -// Relation 33 (MON$DATABASE) - - const USHORT f_mon_db_name = 0; - const USHORT f_mon_db_page_size = 1; - const USHORT f_mon_db_ods_major = 2; - const USHORT f_mon_db_ods_minor = 3; - const USHORT f_mon_db_oit = 4; - const USHORT f_mon_db_oat = 5; - const USHORT f_mon_db_ost = 6; - const USHORT f_mon_db_nt = 7; - const USHORT f_mon_db_page_bufs = 8; - const USHORT f_mon_db_dialect = 9; - const USHORT f_mon_db_shut_mode = 10; - const USHORT f_mon_db_sweep_int = 11; - const USHORT f_mon_db_read_only = 12; - const USHORT f_mon_db_forced_writes = 13; - const USHORT f_mon_db_res_space = 14; - const USHORT f_mon_db_created = 15; - const USHORT f_mon_db_pages = 16; - const USHORT f_mon_db_stat_id = 17; - const USHORT f_mon_db_backup_state = 18; - const USHORT f_mon_db_crypt_page = 19; - const USHORT f_mon_db_owner = 20; - const USHORT f_mon_db_secdb = 21; - const USHORT f_mon_db_crypt_state = 22; - - -// Relation 34 (MON$ATTACHMENTS) - - const USHORT f_mon_att_id = 0; - const USHORT f_mon_att_server_pid = 1; - const USHORT f_mon_att_state = 2; - const USHORT f_mon_att_name = 3; - const USHORT f_mon_att_user = 4; - const USHORT f_mon_att_role = 5; - const USHORT f_mon_att_remote_proto = 6; - const USHORT f_mon_att_remote_addr = 7; - const USHORT f_mon_att_remote_pid = 8; - const USHORT f_mon_att_charset_id = 9; - const USHORT f_mon_att_timestamp = 10; - const USHORT f_mon_att_gc = 11; - const USHORT f_mon_att_remote_process = 12; - const USHORT f_mon_att_stat_id = 13; - const USHORT f_mon_att_client_version = 14; - const USHORT f_mon_att_remote_version = 15; - const USHORT f_mon_att_remote_host = 16; - const USHORT f_mon_att_remote_os_user = 17; - const USHORT f_mon_att_auth_method = 18; - const USHORT f_mon_att_sys_flag = 19; - const USHORT f_mon_att_idle_timeout = 20; - const USHORT f_mon_att_idle_timer = 21; - const USHORT f_mon_att_stmt_timeout = 22; - const USHORT f_mon_att_wire_compressed = 23; - const USHORT f_mon_att_wire_encrypted = 24; - const USHORT f_mon_att_remote_crypt = 25; - - -// Relation 35 (MON$TRANSACTIONS) - - const USHORT f_mon_tra_id = 0; - const USHORT f_mon_tra_att_id = 1; - const USHORT f_mon_tra_state = 2; - const USHORT f_mon_tra_timestamp = 3; - const USHORT f_mon_tra_top = 4; - const USHORT f_mon_tra_oit = 5; - const USHORT f_mon_tra_oat = 6; - const USHORT f_mon_tra_iso_mode = 7; - const USHORT f_mon_tra_lock_timeout = 8; - const USHORT f_mon_tra_read_only = 9; - const USHORT f_mon_tra_auto_commit = 10; - const USHORT f_mon_tra_auto_undo = 11; - const USHORT f_mon_tra_stat_id = 12; - - -// Relation 36 (MON$STATEMENTS) - - const USHORT f_mon_stmt_id = 0; - const USHORT f_mon_stmt_att_id = 1; - const USHORT f_mon_stmt_tra_id = 2; - const USHORT f_mon_stmt_state = 3; - const USHORT f_mon_stmt_timestamp = 4; - const USHORT f_mon_stmt_sql_text = 5; - const USHORT f_mon_stmt_stat_id = 6; - const USHORT f_mon_stmt_expl_plan = 7; - const USHORT f_mon_stmt_timeout = 8; - const USHORT f_mon_stmt_timer = 9; - - -// Relation 37 (MON$CALL_STACK) - - const USHORT f_mon_call_id = 0; - const USHORT f_mon_call_stmt_id = 1; - const USHORT f_mon_call_caller_id = 2; - const USHORT f_mon_call_name = 3; - const USHORT f_mon_call_type = 4; - const USHORT f_mon_call_timestamp = 5; - const USHORT f_mon_call_src_line = 6; - const USHORT f_mon_call_src_column = 7; - const USHORT f_mon_call_stat_id = 8; - const USHORT f_mon_call_pkg_name = 9; - - -// Relation 38 (MON$IO_STATS) - - const USHORT f_mon_io_stat_id = 0; - const USHORT f_mon_io_stat_group = 1; - const USHORT f_mon_io_page_reads = 2; - const USHORT f_mon_io_page_writes = 3; - const USHORT f_mon_io_page_fetches = 4; - const USHORT f_mon_io_page_marks = 5; - - -// Relation 39 (MON$RECORD_STATS) - - const USHORT f_mon_rec_stat_id = 0; - const USHORT f_mon_rec_stat_group = 1; - const USHORT f_mon_rec_seq_reads = 2; - const USHORT f_mon_rec_idx_reads = 3; - const USHORT f_mon_rec_inserts = 4; - const USHORT f_mon_rec_updates = 5; - const USHORT f_mon_rec_deletes = 6; - const USHORT f_mon_rec_backouts = 7; - const USHORT f_mon_rec_purges = 8; - const USHORT f_mon_rec_expunges = 9; - const USHORT f_mon_rec_locks = 10; - const USHORT f_mon_rec_waits = 11; - const USHORT f_mon_rec_conflicts = 12; - const USHORT f_mon_rec_bkver_reads = 13; - const USHORT f_mon_rec_frg_reads = 14; - const USHORT f_mon_rec_rpt_reads = 15; - const USHORT f_mon_rec_imgc = 16; - - -// Relation 40 (MON$CONTEXT_VARIABLES) - - const USHORT f_mon_ctx_var_att_id = 0; - const USHORT f_mon_ctx_var_tra_id = 1; - const USHORT f_mon_ctx_var_name = 2; - const USHORT f_mon_ctx_var_value = 3; - - -// Relation 41 (MON$MEMORY_USAGE) - - const USHORT f_mon_mem_stat_id = 0; - const USHORT f_mon_mem_stat_group = 1; - const USHORT f_mon_mem_cur_used = 2; - const USHORT f_mon_mem_cur_alloc = 3; - const USHORT f_mon_mem_max_used = 4; - const USHORT f_mon_mem_max_alloc = 5; - - -// Relation 42 (RDB$PACKAGES) - - const USHORT f_pkg_name = 0; - const USHORT f_pkg_header_source = 1; - const USHORT f_pkg_body_source = 2; - const USHORT f_pkg_valid_body_flag = 3; - const USHORT f_pkg_class = 4; - const USHORT f_pkg_owner = 5; - const USHORT f_pkg_sys_flag = 6; - const USHORT f_pkg_desc = 7; - const USHORT f_pkg_sql_security = 8; - - -// Relation 43 (SEC$USERS) - - const USHORT f_sec_user_name = 0; - const USHORT f_sec_first_name = 1; - const USHORT f_sec_middle_name = 2; - const USHORT f_sec_last_name = 3; - const USHORT f_sec_active = 4; - const USHORT f_sec_admin = 5; - const USHORT f_sec_comment = 6; - const USHORT f_sec_plugin = 7; - - -// Relation 44 (SEC$USER_ATTRIBUTES) - - const USHORT f_sec_attr_user = 0; - const USHORT f_sec_attr_key = 1; - const USHORT f_sec_attr_value = 2; - const USHORT f_sec_attr_plugin = 3; - - -// Relation 45 (RDB$AUTH_MAPPING) - - const USHORT f_map_name = 0; - const USHORT f_map_using = 1; - const USHORT f_map_plugin = 2; - const USHORT f_map_db = 3; - const USHORT f_map_from_type = 4; - const USHORT f_map_from = 5; - const USHORT f_map_to_type = 6; - const USHORT f_map_to = 7; - const USHORT f_map_sys_flag = 8; - const USHORT f_map_desc = 9; - - -// Relation 46 (SEC$GLOBAL_AUTH_MAPPING) - - const USHORT f_sec_map_name = 0; - const USHORT f_sec_map_using = 1; - const USHORT f_sec_map_plugin = 2; - const USHORT f_sec_map_db = 3; - const USHORT f_sec_map_from_type = 4; - const USHORT f_sec_map_from = 5; - const USHORT f_sec_map_to_type = 6; - const USHORT f_sec_map_to = 7; - - -// Relation 47 (RDB$DB_CREATORS) - - const USHORT f_crt_user = 0; - const USHORT f_crt_u_type = 1; - - -// Relation 48 (SEC$DB_CREATORS) - - const USHORT f_sec_crt_user = 0; - const USHORT f_sec_crt_u_type = 1; - - -// Relation 49 (MON$TABLE_STATS) - - const USHORT f_mon_tab_stat_id = 0; - const USHORT f_mon_tab_stat_group = 1; - const USHORT f_mon_tab_name = 2; - const USHORT f_mon_tab_rec_stat_id = 3; - - -// Relation 50 (RDB$TIME_ZONES) - - const USHORT f_tz_id = 0; - const USHORT f_tz_name = 1; - - diff --git a/src/jrd/DbCreators.cpp b/src/jrd/DbCreators.cpp index 113678912f..ead6f27708 100644 --- a/src/jrd/DbCreators.cpp +++ b/src/jrd/DbCreators.cpp @@ -49,7 +49,7 @@ #include "../jrd/tra.h" #include "../jrd/ini.h" #include "../jrd/status.h" -#include "gen/ids.h" +#include "../jrd/ids.h" #define DBC_DEBUG(A) diff --git a/src/jrd/Mapping.cpp b/src/jrd/Mapping.cpp index 781052398c..a0a9e04da7 100644 --- a/src/jrd/Mapping.cpp +++ b/src/jrd/Mapping.cpp @@ -46,7 +46,7 @@ #include "../jrd/tra.h" #include "../jrd/ini.h" #include "../jrd/status.h" -#include "gen/ids.h" +#include "../jrd/ids.h" #ifdef WIN_NT #include diff --git a/src/jrd/TimeZone.cpp b/src/jrd/TimeZone.cpp index d89c092473..20565c30f0 100644 --- a/src/jrd/TimeZone.cpp +++ b/src/jrd/TimeZone.cpp @@ -25,7 +25,7 @@ #include "../jrd/Record.h" #include "../jrd/ini.h" #include "../jrd/tra.h" -#include "gen/ids.h" +#include "../jrd/ids.h" using namespace Jrd; using namespace Firebird; diff --git a/src/jrd/UserManagement.cpp b/src/jrd/UserManagement.cpp index ccfae63f2d..c1ed05501c 100644 --- a/src/jrd/UserManagement.cpp +++ b/src/jrd/UserManagement.cpp @@ -30,7 +30,7 @@ #include "../common/security.h" #include "../jrd/met_proto.h" #include "../jrd/ini.h" -#include "gen/ids.h" +#include "../jrd/ids.h" using namespace Jrd; using namespace Firebird; diff --git a/src/jrd/ids.h b/src/jrd/ids.h new file mode 100644 index 0000000000..5d240d311d --- /dev/null +++ b/src/jrd/ids.h @@ -0,0 +1,89 @@ +/* + * PROGRAM: JRD Access Method + * MODULE: ids.h + * DESCRIPTION: System relation field numbers + * + * The contents of this file are subject to the Interbase Public + * License Version 1.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy + * of the License at http://www.Inprise.com/IPL.html + * + * Software distributed under the License is distributed on an + * "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express + * or implied. See the License for the specific language governing + * rights and limitations under the License. + * + * The Original Code was created by Inprise Corporation + * and its predecessors. Portions created by Inprise Corporation are + * Copyright (C) Inprise Corporation. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + */ + +#define RELATION(...) enum : USHORT { +#define FIELD(field_id, ...) field_id, +#define END_RELATION }; + +#include "relations.h" + +#undef RELATION +#undef FIELD +#undef END_RELATION + + +// Because it is ODS-related header, an additional check +// to ensure compatibility: position of one field for each table is checked. +// This field don't have to be the last, any one is good. + + static_assert(f_pag_type == 3, "Wrong field id"); + static_assert(f_dat_sql_security == 5, "Wrong field id"); + static_assert(f_fld_owner == 29, "Wrong field id"); + static_assert(f_seg_statistics == 3, "Wrong field id"); + static_assert(f_idx_statistics == 12, "Wrong field id"); + static_assert(f_rfr_identity_type == 20, "Wrong field id"); + static_assert(f_rel_sql_security == 17, "Wrong field id"); + static_assert(f_vrl_pkg_name == 5, "Wrong field id"); + static_assert(f_fmt_desc == 2, "Wrong field id"); + static_assert(f_cls_desc == 2, "Wrong field id"); + static_assert(f_file_shad_num == 5, "Wrong field id"); + static_assert(f_typ_sys_flag == 4, "Wrong field id"); + static_assert(f_trg_sql_security == 14, "Wrong field id"); + static_assert(f_dpd_pkg_name == 5, "Wrong field id"); + static_assert(f_fun_sql_security == 20, "Wrong field id"); + static_assert(f_arg_desc == 21, "Wrong field id"); + static_assert(f_flt_owner == 8, "Wrong field id"); + static_assert(f_msg_msg == 2, "Wrong field id"); + static_assert(f_prv_o_type == 7, "Wrong field id"); + static_assert(f_trn_desc == 3, "Wrong field id"); + static_assert(f_gen_increment == 7, "Wrong field id"); + static_assert(f_dims_upper == 3, "Wrong field id"); + static_assert(f_rcon_iname == 5, "Wrong field id"); + static_assert(f_refc_del_rul == 4, "Wrong field id"); + static_assert(f_ccon_tname == 1, "Wrong field id"); + static_assert(f_log_flags == 5, "Wrong field id"); + static_assert(f_prc_sql_security == 18, "Wrong field id"); + static_assert(f_prm_pkg_name == 14, "Wrong field id"); + static_assert(f_cs_owner == 10, "Wrong field id"); + static_assert(f_coll_owner == 10, "Wrong field id"); + static_assert(f_xcp_owner == 6, "Wrong field id"); + static_assert(f_rol_sys_priv == 5, "Wrong field id"); + static_assert(f_backup_name == 5, "Wrong field id"); + static_assert(f_mon_db_crypt_state == 22, "Wrong field id"); + static_assert(f_mon_att_remote_crypt == 25, "Wrong field id"); + static_assert(f_mon_tra_stat_id == 12, "Wrong field id"); + static_assert(f_mon_stmt_timer == 9, "Wrong field id"); + static_assert(f_mon_call_pkg_name == 9, "Wrong field id"); + static_assert(f_mon_io_page_marks == 5, "Wrong field id"); + static_assert(f_mon_rec_imgc == 16, "Wrong field id"); + static_assert(f_mon_ctx_var_value == 3, "Wrong field id"); + static_assert(f_mon_mem_max_alloc == 5, "Wrong field id"); + static_assert(f_pkg_sql_security == 8, "Wrong field id"); + static_assert(f_sec_plugin == 7, "Wrong field id"); + static_assert(f_sec_attr_plugin == 3, "Wrong field id"); + static_assert(f_map_desc == 9, "Wrong field id"); + static_assert(f_sec_map_to == 7, "Wrong field id"); + static_assert(f_crt_u_type == 1, "Wrong field id"); + static_assert(f_sec_crt_u_type == 1, "Wrong field id"); + static_assert(f_mon_tab_rec_stat_id == 3, "Wrong field id"); + static_assert(f_tz_name == 1, "Wrong field id"); diff --git a/src/jrd/ini.epp b/src/jrd/ini.epp index e501980986..84ddfe306e 100644 --- a/src/jrd/ini.epp +++ b/src/jrd/ini.epp @@ -29,7 +29,7 @@ #include "../jrd/val.h" #include "../jrd/ods.h" #include "../jrd/btr.h" -#include "gen/ids.h" +#include "../jrd/ids.h" #include "../jrd/intl.h" #include "../jrd/tra.h" #include "../jrd/trig.h" diff --git a/src/jrd/replication/Applier.cpp b/src/jrd/replication/Applier.cpp index a326481dc2..1fd6ef3c3e 100644 --- a/src/jrd/replication/Applier.cpp +++ b/src/jrd/replication/Applier.cpp @@ -21,7 +21,7 @@ */ #include "firebird.h" -#include "ids.h" +#include "../ids.h" #include "../jrd/align.h" #include "../jrd/jrd.h" #include "../jrd/blb.h" diff --git a/src/jrd/vio.cpp b/src/jrd/vio.cpp index 6c7ff8e06e..e7b1a5d114 100644 --- a/src/jrd/vio.cpp +++ b/src/jrd/vio.cpp @@ -48,7 +48,7 @@ #include "../jrd/val.h" #include "../jrd/req.h" #include "../jrd/tra.h" -#include "gen/ids.h" +#include "../jrd/ids.h" #include "../jrd/lck.h" #include "../jrd/lls.h" #include "../jrd/scl.h" diff --git a/src/misc/ids.m b/src/misc/ids.m deleted file mode 100644 index 899d658abd..0000000000 --- a/src/misc/ids.m +++ /dev/null @@ -1,30 +0,0 @@ -divert(-1) -/* - * The contents of this file are subject to the Interbase Public - * License Version 1.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy - * of the License at http://www.Inprise.com/IPL.html - * - * Software distributed under the License is distributed on an - * "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express - * or implied. See the License for the specific language governing - * rights and limitations under the License. - * - * The Original Code was created by Inprise Corporation - * and its predecessors. Portions created by Inprise Corporation are - * Copyright (C) Inprise Corporation. - * - * All Rights Reserved. - * Contributor(s): ______________________________________. - */ - -changequote([,]) - -define(RELATION, [define([N], 0)]) -define(FIELD, [[const USHORT ] $1 [=] N[;]define([N], incr(N)) -dnl]) -define(END_RELATION, ) -define(FIELD_ODS8, ) - -divert -include(../src/jrd/relations.h) From 4b6eaa57c3f925468b64ca91b95511cd58e2676e Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Tue, 21 Jan 2020 12:57:31 -0300 Subject: [PATCH 235/274] Move tzdata to extern/icu. --- .github/workflows/tzdata-update.yml | 6 +++--- builds/posix/Makefile.in | 4 ++-- builds/win32/make_icu.bat | 2 +- extern/icu/Readme.txt | 6 +++++- {tzdata => extern/icu/tzdata}/be.zip | Bin {tzdata => extern/icu/tzdata}/le.zip | Bin {tzdata => extern/icu/tzdata}/update.sh | 0 {tzdata => extern/icu/tzdata}/version.txt | 0 8 files changed, 11 insertions(+), 7 deletions(-) rename {tzdata => extern/icu/tzdata}/be.zip (100%) rename {tzdata => extern/icu/tzdata}/le.zip (100%) rename {tzdata => extern/icu/tzdata}/update.sh (100%) rename {tzdata => extern/icu/tzdata}/version.txt (100%) diff --git a/.github/workflows/tzdata-update.yml b/.github/workflows/tzdata-update.yml index 7af28a4b59..08842f048e 100644 --- a/.github/workflows/tzdata-update.yml +++ b/.github/workflows/tzdata-update.yml @@ -19,13 +19,13 @@ jobs: VERSION=`ls /tmp/icu-checkout/tzdata/icunew/ -r1a |head -1` echo Last version: $VERSION - if [ "$VERSION" == "`cat tzdata/version.txt`" ] + if [ "$VERSION" == "`cat extern/icu/tzdata/version.txt`" ] then exit fi - echo $VERSION > tzdata/version.txt - tzdata/update.sh + echo $VERSION > extern/icu/tzdata/version.txt + extern/icu/tzdata/update.sh - name: Create Pull Request uses: peter-evans/create-pull-request@v2 diff --git a/builds/posix/Makefile.in b/builds/posix/Makefile.in index 3f05cd22ac..c194682fc3 100644 --- a/builds/posix/Makefile.in +++ b/builds/posix/Makefile.in @@ -624,8 +624,8 @@ $(FIREBIRD_MSG): $(BUILD_FILE) msg.timestamp tzdata: $(FIREBIRD)/tzdata # FIXME: For big-endian, be.zip must be used. -$(FIREBIRD)/tzdata: $(ROOT)/tzdata/le.zip - unzip -o $(ROOT)/tzdata/le.zip -d $(FIREBIRD)/tzdata +$(FIREBIRD)/tzdata: $(ROOT)/extern/icu/tzdata/le.zip + unzip -o $(ROOT)/extern/icu/tzdata/le.zip -d $(FIREBIRD)/tzdata $(BUILD_FILE): $(BUILD_Objects) $(COMMON_LIB) $(EXE_LINK) $(EXE_LINK_OPTIONS) $(LSB_UNDEF) $^ -o $@ $(FIREBIRD_LIBRARY_LINK) $(LINK_LIBS) $(call LINK_DARWIN_RPATH,..) diff --git a/builds/win32/make_icu.bat b/builds/win32/make_icu.bat index 7eaa3aaf0a..ced0139464 100644 --- a/builds/win32/make_icu.bat +++ b/builds/win32/make_icu.bat @@ -15,7 +15,7 @@ if errorlevel 1 call :ERROR build failed - see make_icu_%FB_TARGET_PLATFORM%.log @echo Extracting tzdata mkdir %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\tzdata -call zipjs.bat unzip -source "%FB_LONG_ROOT_PATH%\tzdata\le.zip" -destination %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\tzdata -keep yes +call zipjs.bat unzip -source "%FB_LONG_ROOT_PATH%\extern\icu\tzdata\le.zip" -destination %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\tzdata -keep yes @goto :EOF diff --git a/extern/icu/Readme.txt b/extern/icu/Readme.txt index 58ad881a06..990fa71721 100644 --- a/extern/icu/Readme.txt +++ b/extern/icu/Readme.txt @@ -2,10 +2,14 @@ icu.exe is a self-extract pre-built (by us) IBM ICU 63.1 library. The sources was downloaded from http://site.icu-project.org/download. -The simple fix for bug ICU-20302 (Windows 7: timezone detection on Windows +The simple fix for bug ICU-20302 (Windows 7: timezone detection on Windows is broken) is applyed, see: https://unicode-org.atlassian.net/browse/ICU-20302 https://github.com/unicode-org/icu/pull/315 The build was done using VS 2017 (15.9). + +--- + +tzdata is automatically updated (pull request created) by GitHub Actions tzdata-update.yml. diff --git a/tzdata/be.zip b/extern/icu/tzdata/be.zip similarity index 100% rename from tzdata/be.zip rename to extern/icu/tzdata/be.zip diff --git a/tzdata/le.zip b/extern/icu/tzdata/le.zip similarity index 100% rename from tzdata/le.zip rename to extern/icu/tzdata/le.zip diff --git a/tzdata/update.sh b/extern/icu/tzdata/update.sh similarity index 100% rename from tzdata/update.sh rename to extern/icu/tzdata/update.sh diff --git a/tzdata/version.txt b/extern/icu/tzdata/version.txt similarity index 100% rename from tzdata/version.txt rename to extern/icu/tzdata/version.txt From 6fa1fffbaa6b68e2c735d57e312e416c6b23459c Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 22 Jan 2020 00:07:05 +0000 Subject: [PATCH 236/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 35093fe8b7..2e838a0e0f 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1737 + FORMAL BUILD NUMBER:1739 */ -#define PRODUCT_VER_STRING "4.0.0.1737" -#define FILE_VER_STRING "WI-T4.0.0.1737" -#define LICENSE_VER_STRING "WI-T4.0.0.1737" -#define FILE_VER_NUMBER 4, 0, 0, 1737 +#define PRODUCT_VER_STRING "4.0.0.1739" +#define FILE_VER_STRING "WI-T4.0.0.1739" +#define LICENSE_VER_STRING "WI-T4.0.0.1739" +#define FILE_VER_NUMBER 4, 0, 0, 1739 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1737" +#define FB_BUILD_NO "1739" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 47d4f35077..b8edfc811e 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1737 +BuildNum=1739 NowAt=`pwd` cd `dirname $0` From 675365675425d416da1757b56f2cac78820a4599 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Tue, 21 Jan 2020 22:09:17 -0300 Subject: [PATCH 237/274] Fixed CORE-6236 - RDB$TIME_ZONE_UTIL package has wrong privilege for PUBLIC. --- src/jrd/ini.epp | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/jrd/ini.epp b/src/jrd/ini.epp index 84ddfe306e..0a6e185121 100644 --- a/src/jrd/ini.epp +++ b/src/jrd/ini.epp @@ -71,9 +71,9 @@ const int FB_MAX_ACL_SIZE = 4096; static void add_index_set(thread_db*); -static void add_security_to_sys_obj(thread_db*, AutoRequest&, AutoRequest&, AutoRequest&, +static void add_security_to_sys_obj(thread_db*, AutoRequest&, AutoRequest&, AutoRequest&, const MetaName&, USHORT, const MetaName&, USHORT = 0, const UCHAR* = NULL); -static void add_security_class(thread_db* tdbb, AutoRequest&, const MetaName& class_name, +static void add_security_class(thread_db* tdbb, AutoRequest&, const MetaName& class_name, USHORT acl_length, const UCHAR* acl); static void add_security_to_sys_rel(thread_db*, AutoRequest&, AutoRequest&, AutoRequest&, const MetaName&, const TEXT*, const USHORT, const UCHAR*); @@ -411,9 +411,12 @@ void INI_format(const char* owner, const char* charset) const UCHAR NON_REL_OWNER_ACL[] = {ACL_priv_list, priv_control, priv_alter, priv_drop, priv_usage, ACL_end}; - const UCHAR NON_REL_PUBLIC_ACL[] = + const UCHAR NON_REL_PUBLIC_USAGE_ACL[] = {ACL_priv_list, priv_usage, ACL_end}; + const UCHAR PKG_PUBLIC_EXECUTE_ACL[] = + {ACL_priv_list, priv_execute, ACL_end}; + UCHAR buffer[FB_MAX_ACL_SIZE]; UCHAR* acl = buffer; *acl++ = ACL_version; @@ -431,8 +434,11 @@ void INI_format(const char* owner, const char* charset) *acl++ = ACL_id_list; *acl++ = ACL_end; - memcpy(acl, NON_REL_PUBLIC_ACL, sizeof(NON_REL_PUBLIC_ACL)); - acl += sizeof(NON_REL_PUBLIC_ACL); + + UCHAR* aclPublicStart = acl; + + memcpy(acl, NON_REL_PUBLIC_USAGE_ACL, sizeof(NON_REL_PUBLIC_USAGE_ACL)); + acl += sizeof(NON_REL_PUBLIC_USAGE_ACL); *acl++ = ACL_end; // Put an extra terminator to avoid scl.epp:walk_acl() missing the end. USHORT length = acl - buffer; @@ -471,13 +477,21 @@ void INI_format(const char* owner, const char* charset) add_security_to_sys_obj(tdbb, reqAddSC, reqModObjSC, reqInsUserPriv, ownerName, obj_collation, collation->name, length, buffer); } + // Must be last! + acl = aclPublicStart; + memcpy(acl, PKG_PUBLIC_EXECUTE_ACL, sizeof(PKG_PUBLIC_EXECUTE_ACL)); + acl += sizeof(PKG_PUBLIC_EXECUTE_ACL); + *acl++ = ACL_end; // Put an extra terminator to avoid scl.epp:walk_acl() missing the end. + length = acl - buffer; + reqModObjSC.reset(); for (auto& systemPackage : SystemPackage::get()) { if (systemPackage.odsVersion > ENCODE_ODS(dbb->dbb_ods_version, dbb->dbb_minor_version)) continue; - add_security_to_sys_obj(tdbb, reqAddSC, reqModObjSC, reqInsUserPriv, ownerName, obj_package_header, systemPackage.name, length, buffer); + add_security_to_sys_obj(tdbb, reqAddSC, reqModObjSC, reqInsUserPriv, ownerName, + obj_package_header, systemPackage.name, length, buffer); } } @@ -1091,8 +1105,8 @@ static void add_security_to_sys_rel(thread_db* tdbb, // Add security to system objects. static void add_security_to_sys_obj(thread_db* tdbb, - AutoRequest& reqAddSC, - AutoRequest& reqModObjSC, + AutoRequest& reqAddSC, + AutoRequest& reqModObjSC, AutoRequest& reqInsUserPriv, const MetaName& user_name, USHORT obj_type, From e6d079293459b6939220a9d86ca396d29a09a028 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Thu, 23 Jan 2020 00:04:37 +0000 Subject: [PATCH 238/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 2e838a0e0f..c6fc0a83f9 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1739 + FORMAL BUILD NUMBER:1740 */ -#define PRODUCT_VER_STRING "4.0.0.1739" -#define FILE_VER_STRING "WI-T4.0.0.1739" -#define LICENSE_VER_STRING "WI-T4.0.0.1739" -#define FILE_VER_NUMBER 4, 0, 0, 1739 +#define PRODUCT_VER_STRING "4.0.0.1740" +#define FILE_VER_STRING "WI-T4.0.0.1740" +#define LICENSE_VER_STRING "WI-T4.0.0.1740" +#define FILE_VER_NUMBER 4, 0, 0, 1740 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1739" +#define FB_BUILD_NO "1740" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index b8edfc811e..c997c6d447 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1739 +BuildNum=1740 NowAt=`pwd` cd `dirname $0` From e93a729892fdb2d8839ebfdbb6a76662fde1b363 Mon Sep 17 00:00:00 2001 From: Dmitry Yemanov Date: Thu, 23 Jan 2020 12:18:44 +0300 Subject: [PATCH 239/274] Postfix for CORE-6236 (RDB package has wrong privilege for PUBLIC) --- src/jrd/ini.epp | 47 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/src/jrd/ini.epp b/src/jrd/ini.epp index 0a6e185121..fb8cf5b87c 100644 --- a/src/jrd/ini.epp +++ b/src/jrd/ini.epp @@ -1123,6 +1123,8 @@ static void add_security_to_sys_obj(thread_db* tdbb, add_security_class(tdbb, reqAddSC, security_class, acl_length, acl); + bool needsUsagePrivileges = false, needsExecPrivileges = false; + if (obj_type == obj_field) { FOR(REQUEST_HANDLE reqModObjSC) FLD IN RDB$FIELDS @@ -1132,6 +1134,8 @@ static void add_security_to_sys_obj(thread_db* tdbb, FLD.RDB$SECURITY_CLASS.NULL = FALSE; PAD(security_class.c_str(), FLD.RDB$SECURITY_CLASS); END_MODIFY + + needsUsagePrivileges = true; } END_FOR } @@ -1144,6 +1148,8 @@ static void add_security_to_sys_obj(thread_db* tdbb, CS.RDB$SECURITY_CLASS.NULL = FALSE; PAD(security_class.c_str(), CS.RDB$SECURITY_CLASS); END_MODIFY + + needsUsagePrivileges = true; } END_FOR } @@ -1156,6 +1162,8 @@ static void add_security_to_sys_obj(thread_db* tdbb, COLL.RDB$SECURITY_CLASS.NULL = FALSE; PAD(security_class.c_str(), COLL.RDB$SECURITY_CLASS); END_MODIFY + + needsUsagePrivileges = true; } END_FOR } @@ -1168,6 +1176,8 @@ static void add_security_to_sys_obj(thread_db* tdbb, XCP.RDB$SECURITY_CLASS.NULL = FALSE; PAD(security_class.c_str(), XCP.RDB$SECURITY_CLASS); END_MODIFY + + needsUsagePrivileges = true; } END_FOR } @@ -1180,6 +1190,8 @@ static void add_security_to_sys_obj(thread_db* tdbb, GEN.RDB$SECURITY_CLASS.NULL = FALSE; PAD(security_class.c_str(), GEN.RDB$SECURITY_CLASS); END_MODIFY + + needsUsagePrivileges = true; } END_FOR } @@ -1192,6 +1204,8 @@ static void add_security_to_sys_obj(thread_db* tdbb, PKG.RDB$SECURITY_CLASS.NULL = FALSE; PAD(security_class.c_str(), PKG.RDB$SECURITY_CLASS); END_MODIFY + + needsExecPrivileges = true; } END_FOR } @@ -1209,18 +1223,29 @@ static void add_security_to_sys_obj(thread_db* tdbb, else fb_assert(false); - for (const char* p = USAGE_PRIVILEGES; *p; ++p) + const char* const privileges = + needsUsagePrivileges ? USAGE_PRIVILEGES : + needsExecPrivileges ? EXEC_PRIVILEGES : + NULL; + + if (privileges) { - STORE(REQUEST_HANDLE reqInsUserPriv) PRIV IN RDB$USER_PRIVILEGES - PAD(user_name.c_str(), PRIV.RDB$USER); - PAD(obj_name.c_str(), PRIV.RDB$RELATION_NAME); - PRIV.RDB$PRIVILEGE[0] = *p; - PRIV.RDB$PRIVILEGE[1] = 0; - PRIV.RDB$GRANT_OPTION = WITH_GRANT_OPTION; - PRIV.RDB$USER_TYPE = obj_user; - PRIV.RDB$OBJECT_TYPE = obj_type; - PRIV.RDB$GRANTOR.NULL = TRUE; - END_STORE + fb_assert(user_name.hasData()); + fb_assert(obj_name.hasData()); + + for (const char* p = privileges; *p; ++p) + { + STORE(REQUEST_HANDLE reqInsUserPriv) PRIV IN RDB$USER_PRIVILEGES + PAD(user_name.c_str(), PRIV.RDB$USER); + PAD(obj_name.c_str(), PRIV.RDB$RELATION_NAME); + PRIV.RDB$PRIVILEGE[0] = *p; + PRIV.RDB$PRIVILEGE[1] = 0; + PRIV.RDB$GRANT_OPTION = WITH_GRANT_OPTION; + PRIV.RDB$USER_TYPE = obj_user; + PRIV.RDB$OBJECT_TYPE = obj_type; + PRIV.RDB$GRANTOR.NULL = TRUE; + END_STORE + } } } From 8060136b7a7e6a51f2881700fe89f6465b504526 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Fri, 24 Jan 2020 00:04:37 +0000 Subject: [PATCH 240/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index c6fc0a83f9..314846e07e 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1740 + FORMAL BUILD NUMBER:1741 */ -#define PRODUCT_VER_STRING "4.0.0.1740" -#define FILE_VER_STRING "WI-T4.0.0.1740" -#define LICENSE_VER_STRING "WI-T4.0.0.1740" -#define FILE_VER_NUMBER 4, 0, 0, 1740 +#define PRODUCT_VER_STRING "4.0.0.1741" +#define FILE_VER_STRING "WI-T4.0.0.1741" +#define LICENSE_VER_STRING "WI-T4.0.0.1741" +#define FILE_VER_NUMBER 4, 0, 0, 1741 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1740" +#define FB_BUILD_NO "1741" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index c997c6d447..e5bbbefbd5 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1740 +BuildNum=1741 NowAt=`pwd` cd `dirname $0` From 95163e7455cc7f20df23db42b6c3a7cf1e76d6ac Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 24 Jan 2020 19:02:52 +0300 Subject: [PATCH 241/274] Fixed CORE-6237: Performance issue - security database connections cache in SRP plugin --- builds/posix/make.shared.variables | 8 +- src/auth/SecDbCache.cpp | 155 +++++++ src/auth/SecDbCache.h | 109 +++++ .../SecureRemotePassword/server/SrpServer.cpp | 283 +++++++----- src/auth/SecurityDatabase/LegacyServer.cpp | 418 ++++++------------ src/auth/SecurityDatabase/LegacyServer.h | 12 - 6 files changed, 578 insertions(+), 407 deletions(-) create mode 100644 src/auth/SecDbCache.cpp create mode 100644 src/auth/SecDbCache.h diff --git a/builds/posix/make.shared.variables b/builds/posix/make.shared.variables index 3588c9c629..86068519bb 100644 --- a/builds/posix/make.shared.variables +++ b/builds/posix/make.shared.variables @@ -52,10 +52,14 @@ YValve_Objects:= $(call dirObjects,yvalve) $(call dirObjects,yvalve/config) AllObjects += $(YValve_Objects) +# Authentication database connections cache +SecDbCache:= $(call makeObjects,auth,SecDbCache.cpp) + + # Remote Remote_Common:= $(call dirObjects,remote) $(call dirObjects,auth/SecureRemotePassword) Remote_Server:= $(call dirObjects,remote/server) $(call dirObjects,auth/SecureRemotePassword/server) \ - $(call makeObjects,jrd/replication,Config.cpp Utils.cpp) + $(call makeObjects,jrd/replication,Config.cpp Utils.cpp) $(SecDbCache) Remote_Client:= $(call dirObjects,remote/client) $(call dirObjects,auth/SecureRemotePassword/client) \ $(call makeObjects,auth/SecurityDatabase,LegacyClient.cpp) \ $(call dirObjects,plugins/crypt/arc4) @@ -177,7 +181,7 @@ AllObjects += $(LEGACY_USERS_MANAGE_Objects) # Legacy authentication on server -LEGACY_AUTH_SERVER_Objects:= $(call makeObjects,auth/SecurityDatabase,LegacyServer.cpp) +LEGACY_AUTH_SERVER_Objects:= $(call makeObjects,auth/SecurityDatabase,LegacyServer.cpp) $(SecDbCache) AllObjects += $(LEGACY_AUTH_SERVER_Objects) diff --git a/src/auth/SecDbCache.cpp b/src/auth/SecDbCache.cpp new file mode 100644 index 0000000000..4308dc4b85 --- /dev/null +++ b/src/auth/SecDbCache.cpp @@ -0,0 +1,155 @@ +/* + * PROGRAM: JRD Access Method + * MODULE: SecDbCache.cpp + * DESCRIPTION: Cached security database connection + * + * The contents of this file are subject to the Interbase Public + * License Version 1.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy + * of the License at http://www.Inprise.com/IPL.html + * + * Software distributed under the License is distributed on an + * "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express + * or implied. See the License for the specific language governing + * rights and limitations under the License. + * + * The Original Code was created by Inprise Corporation + * and its predecessors. Portions created by Inprise Corporation are + * Copyright (C) Inprise Corporation. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + * + * 2003.02.02 Dmitry Yemanov: Implemented cached security database connection + * 2011 - 2020 Alexander Peshkov + */ + +#include "firebird.h" + +#include "../auth/SecDbCache.h" +#include "../common/status.h" +#include "../common/isc_proto.h" + +#include + + +using namespace Firebird; + +namespace Auth { + +void CachedSecurityDatabase::close() +{ + FbLocalStatus s; + TimerInterfacePtr()->start(&s, this, 10 * 1000 * 1000); + if (s->getState() & IStatus::STATE_ERRORS) + handler(); +} + +void CachedSecurityDatabase::handler() +{ + list->handler(this); +} + + +void PluginDatabases::getInstance(IPluginConfig* pluginConfig, RefPtr& instance) +{ + // Determine sec.db name based on existing config + PathName secDbName; + { // config scope + FbLocalStatus s; + RefPtr config(REF_NO_INCR, pluginConfig->getFirebirdConf(&s)); + check(&s); + + static GlobalPtr keys; + unsigned int secDbKey = keys->getKey(config, "SecurityDatabase"); + const char* tmp = config->asString(secDbKey); + if (!tmp) + Arg::Gds(isc_secdb_name).raise(); + + secDbName = tmp; + } + + { // guard scope + MutexLockGuard g(arrayMutex, FB_FUNCTION); + for (unsigned int i = 0; i < dbArray.getCount(); ++i) + { + if (secDbName == dbArray[i]->secureDbName) + { + instance = dbArray[i]; + break; + } + } + + if (!instance) + { + instance = FB_NEW CachedSecurityDatabase(this, secDbName); + instance->addRef(); + secDbName.copyTo(instance->secureDbName, sizeof(instance->secureDbName)); + dbArray.add(instance); + } + } +} + +int PluginDatabases::shutdown() +{ + try + { + MutexLockGuard g(arrayMutex, FB_FUNCTION); + for (unsigned int i = 0; i < dbArray.getCount(); ++i) + { + if (dbArray[i]) + { + FbLocalStatus s; + TimerInterfacePtr()->stop(&s, dbArray[i]); + check(&s); + dbArray[i]->release(); + dbArray[i] = NULL; + } + } + dbArray.clear(); + } + catch (Exception &ex) + { + StaticStatusVector st; + ex.stuffException(st); + const ISC_STATUS* status = st.begin(); + if (status[0] == 1 && status[1] != isc_att_shutdown) + { + iscLogStatus("Legacy security database shutdown", status); + } + + return FB_FAILURE; + } + + return FB_SUCCESS; +} + +void PluginDatabases::handler(CachedSecurityDatabase* tgt) +{ + try + { + MutexLockGuard g(arrayMutex, FB_FUNCTION); + + for (unsigned int i = 0; i < dbArray.getCount(); ++i) + { + if (dbArray[i] == tgt) + { + dbArray.remove(i); + tgt->release(); + break; + } + } + } + catch (Exception &ex) + { + StaticStatusVector st; + ex.stuffException(st); + const ISC_STATUS* status = st.begin(); + if (status[0] == 1 && status[1] != isc_att_shutdown) + { + iscLogStatus("Legacy security database timer handler", status); + } + } +} + +} // namespace Auth diff --git a/src/auth/SecDbCache.h b/src/auth/SecDbCache.h new file mode 100644 index 0000000000..48d3bb5598 --- /dev/null +++ b/src/auth/SecDbCache.h @@ -0,0 +1,109 @@ +/* + * PROGRAM: JRD Access Method + * MODULE: SecDbCache.h + * DESCRIPTION: Cached security database connection + * + * The contents of this file are subject to the Interbase Public + * License Version 1.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy + * of the License at http://www.Inprise.com/IPL.html + * + * Software distributed under the License is distributed on an + * "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express + * or implied. See the License for the specific language governing + * rights and limitations under the License. + * + * The Original Code was created by Inprise Corporation + * and its predecessors. Portions created by Inprise Corporation are + * Copyright (C) Inprise Corporation. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + * + * 2003.02.02 Dmitry Yemanov: Implemented cached security database connection + * 2011 - 2020 Alexander Peshkov + */ + +#ifndef FB_SECDBCACHE_H +#define FB_SECDBCACHE_H + +#include "firebird/Interface.h" +#include "../common/classes/ImplementHelper.h" +#include "../common/classes/fb_string.h" +#include "../common/classes/array.h" +#include "../common/classes/alloc.h" +#include "../common/classes/auto.h" + + +namespace Auth { + +class VSecDb +{ +public: + VSecDb() + { + } + + virtual ~VSecDb() + { + } + + virtual bool lookup(void* inMsg, void* outMsg) = 0; +}; + + +class PluginDatabases; + +class CachedSecurityDatabase FB_FINAL + : public Firebird::RefCntIface > +{ +public: + char secureDbName[MAXPATHLEN + 1]; + + CachedSecurityDatabase(PluginDatabases* l, const Firebird::PathName& nm) + : secDb(nullptr), list(l) + { + nm.copyTo(secureDbName, sizeof secureDbName); + } + + // ITimer implementation + void handler(); + + int release() + { + if (--refCounter == 0) + { + delete this; + return 0; + } + + return 1; + } + + void close(); + + Firebird::Mutex mutex; + Firebird::AutoPtr secDb; + PluginDatabases* list; +}; + +class PluginDatabases +{ +public: + PluginDatabases(MemoryPool& p) + : dbArray(p) + { } + +private: + Firebird::HalfStaticArray dbArray; + Firebird::Mutex arrayMutex; + +public: + void getInstance(Firebird::IPluginConfig* pluginConfig, Firebird::RefPtr& instance); + int shutdown(); + void handler(CachedSecurityDatabase* tgt); +}; + +} // namespace Auth + +#endif // FB_SECDBCACHE_H diff --git a/src/auth/SecureRemotePassword/server/SrpServer.cpp b/src/auth/SecureRemotePassword/server/SrpServer.cpp index df4138ac8d..f05216ef8a 100644 --- a/src/auth/SecureRemotePassword/server/SrpServer.cpp +++ b/src/auth/SecureRemotePassword/server/SrpServer.cpp @@ -25,28 +25,50 @@ */ #include "firebird.h" +#include "firebird/Message.h" -#include "../auth/SecureRemotePassword/client/SrpClient.h" +#include "../auth/SecureRemotePassword/server/SrpServer.h" #include "../auth/SecureRemotePassword/srp.h" #include "../common/classes/ImplementHelper.h" #include "../common/classes/ClumpletWriter.h" -#include "../auth/SecureRemotePassword/Message.h" +#include "../common/status.h" #include "../common/classes/ParsedList.h" +#include "../common/isc_proto.h" #include "../jrd/constants.h" +#include "../auth/SecDbCache.h" using namespace Firebird; +using namespace Auth; namespace { -GlobalPtr keys; +GlobalPtr instances; -const unsigned int SZ_LOGIN = 31; +const unsigned int SZ_LOGIN = 63; -} +struct Metadata +{ + FbLocalStatus status; + FB_MESSAGE (Param, CheckStatusWrapper, + (FB_VARCHAR(SZ_LOGIN), login) + ) param; + FB_MESSAGE (Data, CheckStatusWrapper, + (FB_VARCHAR(128), verifier) + (FB_VARCHAR(32), salt) + ) data; + Metadata() + : param(&status, MasterInterfacePtr()), data(&status, MasterInterfacePtr()) + { } + + Metadata(MemoryPool& p) + : status(p), param(&status, MasterInterfacePtr()), data(&status, MasterInterfacePtr()) + { } +}; + +InitInstance meta; -namespace Auth { class SrpServer : public StdPlugin > { @@ -55,13 +77,8 @@ public: : server(NULL), data(getPool()), account(getPool()), clientPubKey(getPool()), serverPubKey(getPool()), verifier(getPool()), salt(getPool()), sessionKey(getPool()), - secDbName(NULL), cryptCallback(NULL) - { - LocalStatus ls; - CheckStatusWrapper s(&ls); - config.assignRefNoIncr(par->getFirebirdConf(&s)); - check(&s); - } + iParameter(par), secDbName(getPool()), cryptCallback(NULL) + { } // IServer implementation int authenticate(CheckStatusWrapper* status, IServerBlock* sBlock, IWriter* writerInterface); @@ -81,14 +98,108 @@ private: UCharBuffer verifier; string salt; UCharBuffer sessionKey; - RefPtr config; - const char* secDbName; + RefPtr iParameter; + PathName secDbName; ICryptKeyCallback* cryptCallback; protected: virtual RemotePassword* remotePasswordFactory() = 0; }; + +class SecurityDatabase : public VSecDb +{ +public: + bool lookup(void* inMsg, void* outMsg) + { + FbLocalStatus status; + + stmt->execute(&status, tra, meta().param.getMetadata(), inMsg, + meta().data.getMetadata(), outMsg); + check(&status); + + return false; // safe default + } + + // This 2 are needed to satisfy temporarily different calling requirements + static int shutdown(const int, const int, void*) + { + return instances->shutdown(); + } + static void cleanup() + { + instances->shutdown(); + } + + SecurityDatabase(const char* secDbName, ICryptKeyCallback* cryptCallback) + : att(nullptr), tra(nullptr), stmt(nullptr) + { + FbLocalStatus status; + + DispatcherPtr p; + if (cryptCallback) + { + p->setDbCryptCallback(&status, cryptCallback); + status->init(); // ignore possible errors like missing call in provider + } + + ClumpletWriter dpb(ClumpletReader::dpbList, MAX_DPB_SIZE); + dpb.insertByte(isc_dpb_sec_attach, TRUE); + dpb.insertString(isc_dpb_user_name, DBA_USER_NAME, fb_strlen(DBA_USER_NAME)); + dpb.insertString(isc_dpb_config, ParsedList::getNonLoopbackProviders(secDbName)); + att = p->attachDatabase(&status, secDbName, dpb.getBufferLength(), dpb.getBuffer()); + check(&status); + HANDSHAKE_DEBUG(fprintf(stderr, "Srv SRP: attached sec db %s\n", secDbName)); + + const UCHAR tpb[] = + { + isc_tpb_version1, + isc_tpb_read, + isc_tpb_read_committed, + isc_tpb_rec_version, + isc_tpb_wait + }; + tra = att->startTransaction(&status, sizeof(tpb), tpb); + check(&status); + HANDSHAKE_DEBUG(fprintf(stderr, "Srv: SRP1: started transaction\n")); + + const char* sql = + "SELECT PLG$VERIFIER, PLG$SALT FROM PLG$SRP WHERE PLG$USER_NAME = ? AND PLG$ACTIVE"; + stmt = att->prepare(&status, tra, 0, sql, 3, IStatement::PREPARE_PREFETCH_METADATA); + if (status->getState() & IStatus::STATE_ERRORS) + { + checkStatusVectorForMissingTable(status->getErrors()); + status_exception::raise(&status); + } + } + +private: + IAttachment* att; + ITransaction* tra; + IStatement* stmt; + + ~SecurityDatabase() + { + FbLocalStatus status; + + stmt->free(&status); + checkLogStatus(status); + + tra->rollback(&status); + checkLogStatus(status); + + att->detach(&status); + checkLogStatus(status); + } + + void checkLogStatus(FbLocalStatus& status) + { + if (!status.isSuccess()) + iscLogStatus("Srp Server", &status); + } +}; + + template class SrpServerImpl FB_FINAL : public SrpServer { public: @@ -103,6 +214,7 @@ protected: } }; + int SrpServer::authenticate(CheckStatusWrapper* status, IServerBlock* sb, IWriter* writerInterface) { try @@ -129,105 +241,46 @@ int SrpServer::authenticate(CheckStatusWrapper* status, IServerBlock* sb, IWrite return AUTH_MORE_DATA; } - // read salt and verifier from database - // obviously we need something like attachments cache here - unsigned int secDbKey = keys->getKey(config, "SecurityDatabase"); - secDbName = config->asString(secDbKey); - if (!(secDbName && secDbName[0])) - { - Arg::Gds(isc_secdb_name).raise(); - } + // load verifier and salt from security database + Metadata messages; + messages.param->login.set(account.c_str()); + messages.param->loginNull = 0; + messages.data.clear(); - DispatcherPtr p; - IAttachment* att = NULL; - ITransaction* tra = NULL; - IStatement* stmt = NULL; + { // reference & mutex scope scope + // Get database block from cache + RefPtr instance; + instances->getInstance(iParameter, instance); - try - { - if (cryptCallback) + try { - p->setDbCryptCallback(status, cryptCallback); - status->init(); // ignore possible errors like missing call in provider + MutexLockGuard g(instance->mutex, FB_FUNCTION); + + secDbName = instance->secureDbName; + if (!instance->secDb) + instance->secDb = FB_NEW SecurityDatabase(instance->secureDbName, cryptCallback); + + instance->secDb->lookup(messages.param.getData(), messages.data.getData()); + } + catch(const Exception&) + { + instance->close(); + throw; } - ClumpletWriter dpb(ClumpletReader::dpbList, MAX_DPB_SIZE); - dpb.insertByte(isc_dpb_sec_attach, TRUE); - dpb.insertString(isc_dpb_user_name, DBA_USER_NAME, fb_strlen(DBA_USER_NAME)); - dpb.insertString(isc_dpb_config, ParsedList::getNonLoopbackProviders(secDbName)); - att = p->attachDatabase(status, secDbName, dpb.getBufferLength(), dpb.getBuffer()); - check(status); - HANDSHAKE_DEBUG(fprintf(stderr, "Srv SRP: attached sec db %s\n", secDbName)); - - const UCHAR tpb[] = - { - isc_tpb_version1, - isc_tpb_read, - isc_tpb_read_committed, - isc_tpb_rec_version, - isc_tpb_wait - }; - tra = att->startTransaction(status, sizeof(tpb), tpb); - check(status); - HANDSHAKE_DEBUG(fprintf(stderr, "Srv: SRP1: started transaction\n")); - - const char* sql = - "SELECT PLG$VERIFIER, PLG$SALT FROM PLG$SRP WHERE PLG$USER_NAME = ? AND PLG$ACTIVE"; - stmt = att->prepare(status, tra, 0, sql, 3, IStatement::PREPARE_PREFETCH_METADATA); - if (status->getState() & IStatus::STATE_ERRORS) - { - checkStatusVectorForMissingTable(status->getErrors()); - status_exception::raise(status); - } - - Meta im(stmt, false); - Message par(im); - Field login(par); - login = account.c_str(); - - Meta om(stmt, true); - Message dat(om); - check(status); - Field verify(dat); - Field slt(dat); - HANDSHAKE_DEBUG(fprintf(stderr, "Srv: SRP1: Ready to run statement with login '%s'\n", account.c_str())); - - stmt->execute(status, tra, par.getMetadata(), par.getBuffer(), - dat.getMetadata(), dat.getBuffer()); - check(status); - HANDSHAKE_DEBUG(fprintf(stderr, "Srv: SRP1: Executed statement\n")); - - verifier.assign(reinterpret_cast(verify->data), verify->len); - dumpIt("Srv: verifier", verifier); - UCharBuffer s; - s.assign(reinterpret_cast(slt->data), slt->len); - BigInteger(s).getText(salt); - dumpIt("Srv: salt", salt); - - stmt->free(status); - check(status); - stmt = NULL; - - tra->rollback(status); - check(status); - tra = NULL; - - att->detach(status); - check(status); - att = NULL; + instance->close(); } - catch (const Exception&) - { - LocalStatus ls; - CheckStatusWrapper s(&ls); + HANDSHAKE_DEBUG(fprintf(stderr, "Srv: SRP1: Executed statement\n")); - if (stmt) stmt->free(&s); - if (tra) tra->rollback(&s); - if (att) att->detach(&s); + verifier.assign(reinterpret_cast(messages.data->verifier.str), messages.data->verifier.length); + dumpIt("Srv: verifier", verifier); - throw; - } + UCharBuffer s; + s.assign(reinterpret_cast(messages.data->salt.str), messages.data->salt.length); + BigInteger(s).getText(salt); + dumpIt("Srv: salt", salt); + // create SRP-calculating server server = remotePasswordFactory(); server->genServerKey(serverPubKey, verifier); @@ -273,7 +326,7 @@ int SrpServer::authenticate(CheckStatusWrapper* status, IServerBlock* sb, IWrite { return AUTH_FAILED; } - writerInterface->setDb(status, secDbName); + writerInterface->setDb(status, secDbName.c_str()); if (status->getState() & IStatus::STATE_ERRORS) { return AUTH_FAILED; @@ -326,14 +379,16 @@ int SrpServer::release() return 1; } -namespace -{ - SimpleFactory > factory_sha1; - SimpleFactory > factory_sha224; - SimpleFactory > factory_sha256; - SimpleFactory > factory_sha384; - SimpleFactory > factory_sha512; -} +SimpleFactory > factory_sha1; +SimpleFactory > factory_sha224; +SimpleFactory > factory_sha256; +SimpleFactory > factory_sha384; +SimpleFactory > factory_sha512; + +} // anonymous namespace + + +namespace Auth { void registerSrpServer(IPluginManager* iPlugin) { diff --git a/src/auth/SecurityDatabase/LegacyServer.cpp b/src/auth/SecurityDatabase/LegacyServer.cpp index e1a223ed0d..64012fb709 100644 --- a/src/auth/SecurityDatabase/LegacyServer.cpp +++ b/src/auth/SecurityDatabase/LegacyServer.cpp @@ -24,26 +24,22 @@ */ #include "firebird.h" -#include -#include -#include + #include "ibase.h" -#include "../auth/SecurityDatabase/LegacyServer.h" -#include "../auth/SecurityDatabase/LegacyHash.h" -#include "../common/enc_proto.h" -#include "../jrd/err_proto.h" -#include "../yvalve/gds_proto.h" -#include "../common/isc_proto.h" -#include "../jrd/jrd_proto.h" -#include "../jrd/scl.h" -#include "../common/config/config.h" -#include "../common/classes/objects_array.h" -#include "../common/classes/init.h" -#include "../common/classes/ImplementHelper.h" -#include "../common/classes/ParsedList.h" +#include "gen/iberror.h" #include "firebird/Interface.h" +#include "../auth/SecurityDatabase/LegacyServer.h" +#include "../auth/SecurityDatabase/LegacyHash.h" +#include "../auth/SecDbCache.h" #include "../remote/remot_proto.h" +#include "../jrd/constants.h" +#include "../common/enc_proto.h" +#include "../common/status.h" +#include "../common/classes/init.h" +#include "../common/classes/ClumpletWriter.h" + +#include #define PLUG_MODULE 1 @@ -110,6 +106,8 @@ struct user_record SCHAR password[Auth::MAX_LEGACY_PASSWORD_LENGTH + 2]; }; +typedef char user_name[129]; + // Transaction parameter buffer const UCHAR TPB[4] = @@ -124,65 +122,50 @@ const UCHAR TPB[4] = namespace Auth { +GlobalPtr instances; + + class SecurityDatabaseServer FB_FINAL : public StdPlugin > { public: - explicit SecurityDatabaseServer(Firebird::IPluginConfig* p) + explicit SecurityDatabaseServer(IPluginConfig* p) : iParameter(p) { } // IServer implementation - int authenticate(Firebird::CheckStatusWrapper* status, Firebird::IServerBlock* sBlock, - Firebird::IWriter* writerInterface); - void setDbCryptCallback(Firebird::CheckStatusWrapper*, Firebird::ICryptKeyCallback*) { } // ignore + int authenticate(CheckStatusWrapper* status, IServerBlock* sBlock, + IWriter* writerInterface); + void setDbCryptCallback(CheckStatusWrapper*, ICryptKeyCallback*) { } // ignore int release(); private: - Firebird::RefPtr iParameter; + RefPtr iParameter; }; -class SecurityDatabase FB_FINAL : public RefCntIface > + +class SecurityDatabase : public VSecDb { public: - int verify(IWriter* authBlock, IServerBlock* sBlock); + bool lookup(void* inMsg, void* outMsg); // This 2 are needed to satisfy temporarily different calling requirements static int shutdown(const int, const int, void*) { - return shutdown(); + return instances->shutdown(); } static void cleanup() { - shutdown(); + instances->shutdown(); } - static int shutdown(); - - char secureDbName[MAXPATHLEN]; - - SecurityDatabase() + SecurityDatabase(const char* secDbName) : lookup_db(0), lookup_req(0) { - } - - // ITimer implementation - void handler(); - - int release() - { - if (--refCounter == 0) - { - delete this; - return 0; - } - - return 1; + prepare(secDbName); } private: - Firebird::Mutex mutex; - ISC_STATUS_ARRAY status; isc_db_handle lookup_db; @@ -190,8 +173,7 @@ private: ~SecurityDatabase(); - bool lookup_user(const char*, char*); - void prepare(); + void prepare(const char* secDbName); void checkStatus(const char* callName, ISC_STATUS userError = isc_psw_db_error); }; @@ -220,58 +202,8 @@ SecurityDatabase::~SecurityDatabase() } } -bool SecurityDatabase::lookup_user(const char* user_name, char* pwd) -{ - bool found = false; // user found flag - char uname[129]; // user name buffer - user_record user; // user record - // Start by clearing the output data - - if (pwd) - *pwd = '\0'; - - fb_utils::copy_terminate(uname, user_name, sizeof uname); - - MutexLockGuard guard(mutex, FB_FUNCTION); - - // Attach database and compile request - - prepare(); - - // Lookup - - isc_tr_handle lookup_trans = 0; - - isc_start_transaction(status, &lookup_trans, 1, &lookup_db, sizeof(TPB), TPB); - checkStatus("isc_start_transaction", isc_psw_start_trans); - - isc_start_and_send(status, &lookup_req, &lookup_trans, 0, sizeof(uname), uname, 0); - checkStatus("isc_start_and_send"); - - while (true) - { - isc_receive(status, &lookup_req, 1, sizeof(user), &user, 0); - checkStatus("isc_receive"); - - if (!user.flag || status[1]) - break; - - found = true; - - if (pwd) - { - fb_utils::copy_terminate(pwd, user.password, MAX_LEGACY_PASSWORD_LENGTH + 1); - } - } - - isc_rollback_transaction(status, &lookup_trans); - checkStatus("isc_rollback_transaction"); - - return found; -} - -void SecurityDatabase::prepare() +void SecurityDatabase::prepare(const char* secureDbName) { if (lookup_db) { @@ -314,75 +246,6 @@ void SecurityDatabase::prepare() checkStatus("isc_compile_request", isc_psw_attach); } -/****************************************************************************** - * - * Public interface - */ - -int SecurityDatabase::verify(IWriter* authBlock, IServerBlock* sBlock) -{ - const char* user = sBlock->getLogin(); - string login(user ? user : ""); - - unsigned length; - const unsigned char* data = sBlock->getData(&length); - string passwordEnc; - if (data) - { - passwordEnc.assign(data, length); - } - - if (!login.hasData()) - { - return IAuth::AUTH_CONTINUE; - } - if (!passwordEnc.hasData()) - { - return IAuth::AUTH_MORE_DATA; - } - - // Look up the user name in the userinfo database and use the parameters - // found there. This means that another database must be accessed, and - // that means the current context must be saved and restored. - - char pw1[MAX_LEGACY_PASSWORD_LENGTH + 1]; - if (!lookup_user(login.c_str(), pw1)) - { - return IAuth::AUTH_CONTINUE; - } - pw1[MAX_LEGACY_PASSWORD_LENGTH] = 0; - string storedHash(pw1, MAX_LEGACY_PASSWORD_LENGTH); - storedHash.rtrim(); - storedHash.recalculate_length(); - - string newHash; - LegacyHash::hash(newHash, login, passwordEnc, storedHash); - if (newHash != storedHash) - { - bool legacyHash = Config::getLegacyHash(); - if (legacyHash) - { - newHash.resize(MAX_LEGACY_PASSWORD_LENGTH + 2); - ENC_crypt(newHash.begin(), newHash.length(), passwordEnc.c_str(), LEGACY_PASSWORD_SALT); - newHash.recalculate_length(); - newHash.erase(0, 2); - legacyHash = newHash == storedHash; - } - if (!legacyHash) - { - return IAuth::AUTH_CONTINUE; - } - } - - LocalStatus ls; - CheckStatusWrapper s(&ls); - authBlock->add(&s, login.c_str()); - check(&s); - authBlock->setDb(&s, secureDbName); - check(&s); - return IAuth::AUTH_SUCCESS; -} - void SecurityDatabase::checkStatus(const char* callName, ISC_STATUS userError) { if (status[1] == 0) @@ -402,139 +265,136 @@ void SecurityDatabase::checkStatus(const char* callName, ISC_STATUS userError) secDbError.raise(); } -typedef HalfStaticArray InstancesArray; -GlobalPtr instances; -GlobalPtr instancesMutex; - -void SecurityDatabase::handler() +bool SecurityDatabase::lookup(void* inMsg, void* outMsg) { - try - { - MutexLockGuard g(instancesMutex, FB_FUNCTION); + isc_tr_handle lookup_trans = 0; - InstancesArray& curInstances(instances); - for (unsigned int i = 0; i < curInstances.getCount(); ++i) - { - if (curInstances[i] == this) - { - curInstances.remove(i); - release(); - break; - } - } - } - catch (Exception &ex) + isc_start_transaction(status, &lookup_trans, 1, &lookup_db, sizeof(TPB), TPB); + checkStatus("isc_start_transaction", isc_psw_start_trans); + + isc_start_and_send(status, &lookup_req, &lookup_trans, 0, sizeof(user_name), inMsg, 0); + checkStatus("isc_start_and_send"); + + bool found = false; + while (true) { - StaticStatusVector st; - ex.stuffException(st); - const ISC_STATUS* status = st.begin(); - if (status[0] == 1 && status[1] != isc_att_shutdown) - { - iscLogStatus("Legacy security database timer handler", status); - } + user_record* user = static_cast(outMsg); + isc_receive(status, &lookup_req, 1, sizeof(user_record), user, 0); + checkStatus("isc_receive"); + + if (!user->flag || status[1]) + break; + + found = true; } + + isc_rollback_transaction(status, &lookup_trans); + checkStatus("isc_rollback_transaction"); + + return found; } -int SecurityDatabase::shutdown() -{ - try - { - MutexLockGuard g(instancesMutex, FB_FUNCTION); - InstancesArray& curInstances(instances); - for (unsigned int i = 0; i < curInstances.getCount(); ++i) - { - if (curInstances[i]) - { - LocalStatus ls; - CheckStatusWrapper s(&ls); - TimerInterfacePtr()->stop(&s, curInstances[i]); - check(&s); - curInstances[i]->release(); - curInstances[i] = NULL; - } - } - curInstances.clear(); - } - catch (Exception &ex) - { - StaticStatusVector st; - ex.stuffException(st); - const ISC_STATUS* status = st.begin(); - if (status[0] == 1 && status[1] != isc_att_shutdown) - { - iscLogStatus("Legacy security database shutdown", status); - } - return FB_FAILURE; - } +/****************************************************************************** + * + * Public interface + */ - return FB_SUCCESS; -} - -static Firebird::GlobalPtr keys; - -int SecurityDatabaseServer::authenticate(Firebird::CheckStatusWrapper* status, IServerBlock* sBlock, - IWriter* writerInterface) +int SecurityDatabaseServer::authenticate(CheckStatusWrapper* status, IServerBlock* sBlock, + IWriter* authBlock) { status->init(); try { - PathName secDbName; - { // config scope - LocalStatus ls; - CheckStatusWrapper s(&ls); - RefPtr config(REF_NO_INCR, iParameter->getFirebirdConf(&s)); - check(&s); + const char* user = sBlock->getLogin(); + if (!user) + { + HANDSHAKE_DEBUG(fprintf(stderr, "LegacyServer (nologin) %d\n", IAuth::AUTH_CONTINUE)); + return IAuth::AUTH_CONTINUE; + } + string login(user); - unsigned int secDbKey = keys->getKey(config, "SecurityDatabase"); - const char* tmp = config->asString(secDbKey); - if (!tmp) - { - Arg::Gds(isc_secdb_name).raise(); - } - - secDbName = tmp; + unsigned length; + const unsigned char* data = sBlock->getData(&length); + if (!(data && length)) + { + HANDSHAKE_DEBUG(fprintf(stderr, "LegacyServer (nopw) %d\n", IAuth::AUTH_MORE_DATA)); + return IAuth::AUTH_MORE_DATA; } - RefPtr instance; + bool found = false; + char pw1[MAX_LEGACY_PASSWORD_LENGTH + 1]; + PathName secureDbName; + { // reference & mutex scope scope + // Get database block from cache + RefPtr instance; + instances->getInstance(iParameter, instance); - { // guard scope - MutexLockGuard g(instancesMutex, FB_FUNCTION); - InstancesArray& curInstances(instances); - for (unsigned int i = 0; i < curInstances.getCount(); ++i) + try { - if (secDbName == curInstances[i]->secureDbName) - { - instance = curInstances[i]; - break; - } + MutexLockGuard g(instance->mutex, FB_FUNCTION); + + secureDbName = instance->secureDbName; + if (!instance->secDb) + instance->secDb = FB_NEW SecurityDatabase(instance->secureDbName); + + user_name uname; // user name buffer + login.copyTo(uname, sizeof uname); + user_record user_block; // user record + found = instance->secDb->lookup(uname, &user_block); + fb_utils::copy_terminate(pw1, user_block.password, MAX_LEGACY_PASSWORD_LENGTH + 1); + } + catch(const Exception&) + { + instance->close(); + throw; } - if (!instance) + instance->close(); + } + if (!found) + { + HANDSHAKE_DEBUG(fprintf(stderr, "LegacyServer (badlogin) %d\n", IAuth::AUTH_CONTINUE)); + return IAuth::AUTH_CONTINUE; + } + + string storedHash(pw1, MAX_LEGACY_PASSWORD_LENGTH); + storedHash.rtrim(); + storedHash.recalculate_length(); + + string passwordEnc; + passwordEnc.assign(data, length); + + string newHash; + LegacyHash::hash(newHash, login, passwordEnc, storedHash); + if (newHash != storedHash) + { + bool legacyHash = Config::getLegacyHash(); + if (legacyHash) { - instance = FB_NEW SecurityDatabase; - instance->addRef(); - secDbName.copyTo(instance->secureDbName, sizeof(instance->secureDbName)); - curInstances.add(instance); + newHash.resize(MAX_LEGACY_PASSWORD_LENGTH + 2); + ENC_crypt(newHash.begin(), newHash.length(), passwordEnc.c_str(), LEGACY_PASSWORD_SALT); + newHash.recalculate_length(); + newHash.erase(0, 2); + legacyHash = newHash == storedHash; + } + if (!legacyHash) + { + HANDSHAKE_DEBUG(fprintf(stderr, "LegacyServer (badpw) %d\n", IAuth::AUTH_CONTINUE)); + return IAuth::AUTH_CONTINUE; } } - int rc = instance->verify(writerInterface, sBlock); -#define USE_ATT_RQ_CACHE -#ifdef USE_ATT_RQ_CACHE - LocalStatus ls; - CheckStatusWrapper s(&ls); - TimerInterfacePtr()->start(&s, instance, 10 * 1000 * 1000); - if (s.getState() & IStatus::STATE_ERRORS) - instance->handler(); -#else - instance->handler(); -#endif - HANDSHAKE_DEBUG(fprintf(stderr, "LegacyServer: verify=%d\n", rc)); - return rc; + FbLocalStatus s; + authBlock->add(&s, login.c_str()); + check(&s); + authBlock->setDb(&s, secureDbName.c_str()); + check(&s); + HANDSHAKE_DEBUG(fprintf(stderr, "LegacyServer (OK) %d\n", IAuth::AUTH_SUCCESS)); + return IAuth::AUTH_SUCCESS; } - catch (const Firebird::Exception& ex) + catch (const Exception& ex) { ex.stuffException(status); HANDSHAKE_DEBUG(fprintf(stderr, "LegacyServer: exception status:\n")); @@ -556,12 +416,12 @@ int SecurityDatabaseServer::release() } namespace { - Firebird::SimpleFactory factory; + SimpleFactory factory; } -void registerLegacyServer(Firebird::IPluginManager* iPlugin) +void registerLegacyServer(IPluginManager* iPlugin) { - iPlugin->registerPluginFactory(Firebird::IPluginManager::TYPE_AUTH_SERVER, + iPlugin->registerPluginFactory(IPluginManager::TYPE_AUTH_SERVER, "Legacy_Auth", &factory); } diff --git a/src/auth/SecurityDatabase/LegacyServer.h b/src/auth/SecurityDatabase/LegacyServer.h index 7ab7683aba..43f7f306de 100644 --- a/src/auth/SecurityDatabase/LegacyServer.h +++ b/src/auth/SecurityDatabase/LegacyServer.h @@ -27,20 +27,8 @@ #ifndef AUTH_LEGACY_SERVER_H #define AUTH_LEGACY_SERVER_H -#include "ibase.h" -#include "../common/utils_proto.h" -#include "../common/sha.h" -#include "gen/iberror.h" -#include "../common/classes/ClumpletWriter.h" -#include "../common/classes/ImplementHelper.h" - #include "firebird/Interface.h" -#ifdef HAVE_STDLIB_H -#include -#endif -#include - namespace Auth { void registerLegacyServer(Firebird::IPluginManager* iPlugin); From 98efe88f11cb167633233565f74795ecaa927e19 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sun, 26 Jan 2020 12:23:42 -0300 Subject: [PATCH 242/274] Fixed CORE-6233 - Wrong dependencies of stored function on view after backup and restore. --- src/burp/restore.epp | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/burp/restore.epp b/src/burp/restore.epp index 0cce0e69f0..267119b87c 100644 --- a/src/burp/restore.epp +++ b/src/burp/restore.epp @@ -4903,12 +4903,15 @@ bool get_function(BurpGlobals* tdgbl) bool existFlag = false; + Firebird::ITransaction* local_trans = tdgbl->global_trans ? tdgbl->global_trans : gds_trans; + if (tdgbl->runtimeODS >= DB_VERSION_DDL12) { GDS_NAME function_name; bool securityClass = false; - STORE (REQUEST_HANDLE tdgbl->handles_get_function_req_handle1) + STORE (TRANSACTION_HANDLE local_trans + REQUEST_HANDLE tdgbl->handles_get_function_req_handle1) X IN RDB$FUNCTIONS X.RDB$DESCRIPTION.NULL = TRUE; X.RDB$ENGINE_NAME.NULL = TRUE; @@ -4959,12 +4962,12 @@ bool get_function(BurpGlobals* tdgbl) case att_function_description: X.RDB$DESCRIPTION.NULL = FALSE; - get_misc_blob (tdgbl, X.RDB$DESCRIPTION, false); + get_misc_blob (tdgbl, X.RDB$DESCRIPTION, true); break; case att_function_description2: X.RDB$DESCRIPTION.NULL = FALSE; - get_source_blob (tdgbl, X.RDB$DESCRIPTION, false); + get_source_blob (tdgbl, X.RDB$DESCRIPTION, true); break; case att_function_module_name: @@ -5024,7 +5027,7 @@ bool get_function(BurpGlobals* tdgbl) case att_function_blr: if (tdgbl->RESTORE_format >= 10) { - get_blr_blob(tdgbl, X.RDB$FUNCTION_BLR, false); + get_blr_blob(tdgbl, X.RDB$FUNCTION_BLR, true); X.RDB$FUNCTION_BLR.NULL = FALSE; } else @@ -5034,7 +5037,7 @@ bool get_function(BurpGlobals* tdgbl) case att_function_source: if (tdgbl->RESTORE_format >= 10) { - get_source_blob(tdgbl, X.RDB$FUNCTION_SOURCE, false); + get_source_blob(tdgbl, X.RDB$FUNCTION_SOURCE, true); X.RDB$FUNCTION_SOURCE.NULL = FALSE; } else @@ -5054,7 +5057,7 @@ bool get_function(BurpGlobals* tdgbl) case att_function_debug_info: if (tdgbl->RESTORE_format >= 10) { - get_misc_blob(tdgbl, X.RDB$DEBUG_INFO, false); + get_misc_blob(tdgbl, X.RDB$DEBUG_INFO, true); X.RDB$DEBUG_INFO.NULL = FALSE; } else @@ -5133,7 +5136,8 @@ bool get_function(BurpGlobals* tdgbl) } else { - STORE (REQUEST_HANDLE tdgbl->handles_get_function_req_handle1) + STORE (TRANSACTION_HANDLE local_trans + REQUEST_HANDLE tdgbl->handles_get_function_req_handle1) X IN RDB$FUNCTIONS X.RDB$SYSTEM_FLAG = 0; X.RDB$SYSTEM_FLAG.NULL = FALSE; @@ -5279,6 +5283,8 @@ void get_function_arg(BurpGlobals* tdgbl, bool skip_arguments) att_type attribute; scan_attr_t scan_next_attr; + Firebird::ITransaction* local_trans = tdgbl->global_trans ? tdgbl->global_trans : gds_trans; + if (skip_arguments) { skip_init(&scan_next_attr); @@ -5378,7 +5384,8 @@ void get_function_arg(BurpGlobals* tdgbl, bool skip_arguments) if (tdgbl->runtimeODS >= DB_VERSION_DDL12) { // with RDB$FIELD_PRECISION - STORE (REQUEST_HANDLE tdgbl->handles_get_function_arg_req_handle1) + STORE (TRANSACTION_HANDLE local_trans + REQUEST_HANDLE tdgbl->handles_get_function_arg_req_handle1) X IN RDB$FUNCTION_ARGUMENTS X.RDB$FIELD_SUB_TYPE.NULL = TRUE; X.RDB$CHARACTER_SET_ID.NULL = TRUE; @@ -5496,7 +5503,7 @@ void get_function_arg(BurpGlobals* tdgbl, bool skip_arguments) case att_functionarg_default_value: if (tdgbl->RESTORE_format >= 10) { - get_blr_blob(tdgbl, X.RDB$DEFAULT_VALUE, false); + get_blr_blob(tdgbl, X.RDB$DEFAULT_VALUE, true); X.RDB$DEFAULT_VALUE.NULL = FALSE; } else @@ -5587,7 +5594,8 @@ void get_function_arg(BurpGlobals* tdgbl, bool skip_arguments) else if (tdgbl->runtimeODS >= DB_VERSION_DDL10) { // with RDB$FIELD_PRECISION - STORE (REQUEST_HANDLE tdgbl->handles_get_function_arg_req_handle1) + STORE (TRANSACTION_HANDLE local_trans + REQUEST_HANDLE tdgbl->handles_get_function_arg_req_handle1) X IN RDB$FUNCTION_ARGUMENTS X.RDB$FIELD_SUB_TYPE.NULL = TRUE; X.RDB$CHARACTER_SET_ID.NULL = TRUE; @@ -5700,7 +5708,8 @@ void get_function_arg(BurpGlobals* tdgbl, bool skip_arguments) else { // without RDB$FIELD_PRECISION - STORE (REQUEST_HANDLE tdgbl->handles_get_function_arg_req_handle1) + STORE (TRANSACTION_HANDLE local_trans + REQUEST_HANDLE tdgbl->handles_get_function_arg_req_handle1) X IN RDB$FUNCTION_ARGUMENTS X.RDB$FIELD_SUB_TYPE.NULL = TRUE; X.RDB$CHARACTER_SET_ID.NULL = TRUE; From f516072ac1f557f245e5c4bbb3851399056d7f02 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Sun, 26 Jan 2020 12:25:12 -0300 Subject: [PATCH 243/274] Register dependencies only when "csb->csb_g_flags & csb_get_dependencies". --- src/dsql/ExprNodes.cpp | 13 ++++++++----- src/dsql/StmtNodes.cpp | 23 +++++++++++++++-------- src/jrd/par.cpp | 25 +++++++++++++++++-------- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/src/dsql/ExprNodes.cpp b/src/dsql/ExprNodes.cpp index d645e92d94..decd7fc757 100644 --- a/src/dsql/ExprNodes.cpp +++ b/src/dsql/ExprNodes.cpp @@ -3314,7 +3314,7 @@ DmlNode* CastNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* csb if (itemInfo.isSpecial()) node->itemInfo = FB_NEW_POOL(*tdbb->getDefaultPool()) ItemInfo(*tdbb->getDefaultPool(), itemInfo); - if (itemInfo.explicitCollation) + if ((csb->csb_g_flags & csb_get_dependencies) && itemInfo.explicitCollation) { CompilerScratch::Dependency dependency(obj_collation); dependency.number = INTL_TEXT_TYPE(node->castDesc); @@ -4805,10 +4805,13 @@ DmlNode* DefaultNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* csb->csb_blr_reader.getMetaName(relationName); csb->csb_blr_reader.getMetaName(fieldName); - CompilerScratch::Dependency dependency(obj_relation); - dependency.relation = MET_lookup_relation(tdbb, relationName); - dependency.subName = FB_NEW_POOL(pool) MetaName(fieldName); - csb->csb_dependencies.push(dependency); + if (csb->csb_g_flags & csb_get_dependencies) + { + CompilerScratch::Dependency dependency(obj_relation); + dependency.relation = MET_lookup_relation(tdbb, relationName); + dependency.subName = FB_NEW_POOL(pool) MetaName(fieldName); + csb->csb_dependencies.push(dependency); + } jrd_fld* fld = NULL; diff --git a/src/dsql/StmtNodes.cpp b/src/dsql/StmtNodes.cpp index 2e9d79e026..d9c8a98763 100644 --- a/src/dsql/StmtNodes.cpp +++ b/src/dsql/StmtNodes.cpp @@ -2174,7 +2174,7 @@ DmlNode* DeclareVariableNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerS csb->csb_map_item_info.put(Item(Item::TYPE_VARIABLE, node->varId), itemInfo); } - if (itemInfo.explicitCollation) + if ((csb->csb_g_flags & csb_get_dependencies) && itemInfo.explicitCollation) { CompilerScratch::Dependency dependency(obj_collation); dependency.number = INTL_TEXT_TYPE(node->varDesc); @@ -2715,9 +2715,13 @@ DmlNode* ErrorHandlerNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScra if (!MET_load_exception(tdbb, item)) PAR_error(csb, Arg::Gds(isc_xcpnotdef) << item.name); - CompilerScratch::Dependency dependency(obj_exception); - dependency.number = item.code; - csb->csb_dependencies.push(dependency); + if (csb->csb_g_flags & csb_get_dependencies) + { + CompilerScratch::Dependency dependency(obj_exception); + dependency.number = item.code; + csb->csb_dependencies.push(dependency); + } + break; } @@ -2886,7 +2890,7 @@ DmlNode* ExecProcedureNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScr PAR_procedure_parms(tdbb, csb, procedure, node->outputMessage.getAddress(), node->outputSources.getAddress(), node->outputTargets.getAddress(), false); - if (!procedure->isSubRoutine()) + if ((csb->csb_g_flags & csb_get_dependencies) && !procedure->isSubRoutine()) { CompilerScratch::Dependency dependency(obj_procedure); dependency.procedure = procedure; @@ -4500,9 +4504,12 @@ DmlNode* ExceptionNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch if (!MET_load_exception(tdbb, *item)) PAR_error(csb, Arg::Gds(isc_xcpnotdef) << item->name); - CompilerScratch::Dependency dependency(obj_exception); - dependency.number = item->code; - csb->csb_dependencies.push(dependency); + if (csb->csb_g_flags & csb_get_dependencies) + { + CompilerScratch::Dependency dependency(obj_exception); + dependency.number = item->code; + csb->csb_dependencies.push(dependency); + } } break; diff --git a/src/jrd/par.cpp b/src/jrd/par.cpp index a0eabbe1bf..951cdeec40 100644 --- a/src/jrd/par.cpp +++ b/src/jrd/par.cpp @@ -506,9 +506,12 @@ USHORT PAR_desc(thread_db* tdbb, CompilerScratch* csb, dsc* desc, ItemInfo* item } } - CompilerScratch::Dependency dependency(obj_field); - dependency.name = name; - csb->csb_dependencies.push(dependency); + if (csb->csb_g_flags & csb_get_dependencies) + { + CompilerScratch::Dependency dependency(obj_field); + dependency.name = name; + csb->csb_dependencies.push(dependency); + } break; } @@ -568,10 +571,13 @@ USHORT PAR_desc(thread_db* tdbb, CompilerScratch* csb, dsc* desc, ItemInfo* item } } - CompilerScratch::Dependency dependency(obj_relation); - dependency.relation = MET_lookup_relation(tdbb, *relationName); - dependency.subName = fieldName; - csb->csb_dependencies.push(dependency); + if (csb->csb_g_flags & csb_get_dependencies) + { + CompilerScratch::Dependency dependency(obj_relation); + dependency.relation = MET_lookup_relation(tdbb, *relationName); + dependency.subName = fieldName; + csb->csb_dependencies.push(dependency); + } break; } @@ -582,7 +588,7 @@ USHORT PAR_desc(thread_db* tdbb, CompilerScratch* csb, dsc* desc, ItemInfo* item break; } - if (desc->getTextType() != CS_NONE) + if ((csb->csb_g_flags & csb_get_dependencies) && desc->getTextType() != CS_NONE) { CompilerScratch::Dependency dependency(obj_collation); dependency.number = INTL_TEXT_TYPE(*desc); @@ -896,6 +902,9 @@ void PAR_dependency(thread_db* tdbb, CompilerScratch* csb, StreamType stream, SS **************************************/ SET_TDBB(tdbb); + if (!(csb->csb_g_flags & csb_get_dependencies)) + return; + CompilerScratch::Dependency dependency(0); if (csb->csb_rpt[stream].csb_relation) From b5c9f2e13274cc02a8d699000b5a518daf2a6a96 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Mon, 27 Jan 2020 00:04:36 +0000 Subject: [PATCH 244/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 314846e07e..46d566cf15 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1741 + FORMAL BUILD NUMBER:1743 */ -#define PRODUCT_VER_STRING "4.0.0.1741" -#define FILE_VER_STRING "WI-T4.0.0.1741" -#define LICENSE_VER_STRING "WI-T4.0.0.1741" -#define FILE_VER_NUMBER 4, 0, 0, 1741 +#define PRODUCT_VER_STRING "4.0.0.1743" +#define FILE_VER_STRING "WI-T4.0.0.1743" +#define LICENSE_VER_STRING "WI-T4.0.0.1743" +#define FILE_VER_NUMBER 4, 0, 0, 1743 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1741" +#define FB_BUILD_NO "1743" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index e5bbbefbd5..53739c0de7 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1741 +BuildNum=1743 NowAt=`pwd` cd `dirname $0` From f7c90dcd46b7a61a26a28f15d6615d1891f0a5e0 Mon Sep 17 00:00:00 2001 From: Roman Simakov Date: Tue, 28 Jan 2020 15:23:38 +0300 Subject: [PATCH 245/274] Now checkPermission returns void. Reverted forgotten error message. --- src/dsql/DdlNodes.epp | 133 +++++++++++++------------------------- src/dsql/DdlNodes.h | 83 ++++++++++++------------ src/dsql/Nodes.h | 2 +- src/dsql/PackageNodes.epp | 14 ++-- src/dsql/PackageNodes.h | 8 +-- src/include/gen/msgs.h | 2 +- src/jrd/jrd.h | 8 +-- src/jrd/scl.epp | 3 +- src/msgs/messages2.sql | 2 +- 9 files changed, 103 insertions(+), 152 deletions(-) diff --git a/src/dsql/DdlNodes.epp b/src/dsql/DdlNodes.epp index ff24b296e9..c254a2a749 100644 --- a/src/dsql/DdlNodes.epp +++ b/src/dsql/DdlNodes.epp @@ -1109,10 +1109,9 @@ string AlterCharSetNode::internalPrint(NodePrinter& printer) const return "AlterCharSetNode"; } -bool AlterCharSetNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void AlterCharSetNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_charset(tdbb, charSet, SCL_alter); - return true; } void AlterCharSetNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -1173,12 +1172,10 @@ void AlterCharSetNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch //---------------------- -bool AlterEDSPoolSetNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void AlterEDSPoolSetNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { if (!tdbb->getAttachment()->locksmith(tdbb, MODIFY_EXT_CONN_POOL)) status_exception::raise(Arg::Gds(isc_miss_prvlg) << "MODIFY_EXT_CONN_POOL"); - - return true; } string AlterEDSPoolSetNode::internalPrint(NodePrinter& printer) const @@ -1213,12 +1210,10 @@ void AlterEDSPoolSetNode::execute(thread_db* tdbb, DsqlCompilerScratch* /*dsqlSc //---------------------- -bool AlterEDSPoolClearNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void AlterEDSPoolClearNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { if (!tdbb->getAttachment()->locksmith(tdbb, MODIFY_EXT_CONN_POOL)) status_exception::raise(Arg::Gds(isc_miss_prvlg) << "MODIFY_EXT_CONN_POOL"); - - return true; } string AlterEDSPoolClearNode::internalPrint(NodePrinter& printer) const @@ -1273,7 +1268,7 @@ string CommentOnNode::internalPrint(NodePrinter& printer) const return "CommentOnNode"; } -bool CommentOnNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CommentOnNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { Attachment* const attachment = transaction->tra_attachment; @@ -1411,8 +1406,6 @@ bool CommentOnNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) default: fb_assert(false); } - - return true; } // select rdb$relation_name from rdb$relation_fields where rdb$field_name = 'RDB$DESCRIPTION'; @@ -1711,18 +1704,17 @@ DdlNode* CreateAlterFunctionNode::dsqlPass(DsqlCompilerScratch* dsqlScratch) return DdlNode::dsqlPass(dsqlScratch); } -bool CreateAlterFunctionNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateAlterFunctionNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { dsc dscName; dscName.makeText(name.length(), CS_METADATA, (UCHAR*) name.c_str()); if (alter) { if (SCL_check_function(tdbb, &dscName, SCL_alter) || !create) - return true; + return; } SCL_check_create_access(tdbb, SCL_object_function); - return true; } void CreateAlterFunctionNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -2378,12 +2370,11 @@ string AlterExternalFunctionNode::internalPrint(NodePrinter& printer) const return "AlterExternalFunctionNode"; } -bool AlterExternalFunctionNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void AlterExternalFunctionNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { dsc dscName; dscName.makeText(name.length(), CS_METADATA, (UCHAR*) name.c_str()); SCL_check_function(tdbb, &dscName, SCL_alter); - return true; } // Allow changing the entry point and/or the module name of a UDF. @@ -2510,12 +2501,11 @@ DdlNode* DropFunctionNode::dsqlPass(DsqlCompilerScratch* dsqlScratch) return DdlNode::dsqlPass(dsqlScratch); } -bool DropFunctionNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropFunctionNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { dsc dscName; dscName.makeText(name.length(), CS_METADATA, (UCHAR*) name.c_str()); SCL_check_function(tdbb, &dscName, SCL_drop); - return true; } void DropFunctionNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -2724,18 +2714,17 @@ DdlNode* CreateAlterProcedureNode::dsqlPass(DsqlCompilerScratch* dsqlScratch) return DdlNode::dsqlPass(dsqlScratch); } -bool CreateAlterProcedureNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateAlterProcedureNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { dsc dscName; dscName.makeText(name.length(), CS_METADATA, (UCHAR*) name.c_str()); if (alter) { if (SCL_check_procedure(tdbb, &dscName, SCL_alter) || !create) - return true; + return; } SCL_check_create_access(tdbb, SCL_object_procedure); - return true; } void CreateAlterProcedureNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -3315,12 +3304,11 @@ DdlNode* DropProcedureNode::dsqlPass(DsqlCompilerScratch* dsqlScratch) return DdlNode::dsqlPass(dsqlScratch); } -bool DropProcedureNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropProcedureNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { dsc dscName; dscName.makeText(name.length(), CS_METADATA, (UCHAR*) name.c_str()); SCL_check_procedure(tdbb, &dscName, SCL_drop); - return true; } void DropProcedureNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -3599,7 +3587,7 @@ DdlNode* CreateAlterTriggerNode::dsqlPass(DsqlCompilerScratch* dsqlScratch) return DdlNode::dsqlPass(dsqlScratch); } -bool CreateAlterTriggerNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateAlterTriggerNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { if (!create) { @@ -3629,8 +3617,6 @@ bool CreateAlterTriggerNode::checkPermission(thread_db* tdbb, jrd_tra* transacti } else SCL_check_database(tdbb, SCL_alter); - - return true; } void CreateAlterTriggerNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -3795,7 +3781,7 @@ DdlNode* DropTriggerNode::dsqlPass(DsqlCompilerScratch* dsqlScratch) return DdlNode::dsqlPass(dsqlScratch); } -bool DropTriggerNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropTriggerNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { MetaName relationName = getTriggerRelationName(tdbb, transaction, name); @@ -3807,8 +3793,6 @@ bool DropTriggerNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) dscName.makeText(relationName.length(), CS_METADATA, (UCHAR*) relationName.c_str()); SCL_check_relation(tdbb, &dscName, SCL_alter); } - - return true; } void DropTriggerNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -3939,10 +3923,9 @@ string CreateCollationNode::internalPrint(NodePrinter& printer) const return "CreateCollationNode"; } -bool CreateCollationNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateCollationNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_create_access(tdbb, SCL_object_collation); - return true; } void CreateCollationNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -4159,10 +4142,9 @@ string DropCollationNode::internalPrint(NodePrinter& printer) const return "DropCollationNode"; } -bool DropCollationNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropCollationNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_collation(tdbb, name, SCL_drop); - return true; } void DropCollationNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -4319,10 +4301,9 @@ string CreateDomainNode::internalPrint(NodePrinter& printer) const return "CreateDomainNode"; } -bool CreateDomainNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateDomainNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_create_access(tdbb, SCL_object_domain); - return true; } void CreateDomainNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -4973,10 +4954,9 @@ string AlterDomainNode::internalPrint(NodePrinter& printer) const return "AlterDomainNode"; } -bool AlterDomainNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void AlterDomainNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_domain(tdbb, name, SCL_alter); - return true; } void AlterDomainNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -5331,10 +5311,9 @@ string DropDomainNode::internalPrint(NodePrinter& printer) const return "DropDomainNode"; } -bool DropDomainNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropDomainNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_domain(tdbb, name, SCL_drop); - return true; } void DropDomainNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -5464,16 +5443,15 @@ string CreateAlterExceptionNode::internalPrint(NodePrinter& printer) const return "CreateAlterExceptionNode"; } -bool CreateAlterExceptionNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateAlterExceptionNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { if (alter) { if (SCL_check_exception(tdbb, name, SCL_alter) || !create) - return true; + return; } SCL_check_create_access(tdbb, SCL_object_exception); - return true; } void CreateAlterExceptionNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -5608,10 +5586,9 @@ string DropExceptionNode::internalPrint(NodePrinter& printer) const return "DropExceptionNode"; } -bool DropExceptionNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropExceptionNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_exception(tdbb, name, SCL_drop); - return true; } void DropExceptionNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -5683,16 +5660,15 @@ string CreateAlterSequenceNode::internalPrint(NodePrinter& printer) const return "CreateAlterSequenceNode"; } -bool CreateAlterSequenceNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateAlterSequenceNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { if (alter) { if (SCL_check_generator(tdbb, name, SCL_alter) || !create) - return true; + return; } SCL_check_create_access(tdbb, SCL_object_generator); - return true; } void CreateAlterSequenceNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -5959,10 +5935,9 @@ string DropSequenceNode::internalPrint(NodePrinter& printer) const return "DropSequenceNode"; } -bool DropSequenceNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropSequenceNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_generator(tdbb, name, SCL_drop); - return true; } void DropSequenceNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction) @@ -7402,10 +7377,9 @@ string CreateRelationNode::internalPrint(NodePrinter& printer) const return "CreateRelationNode"; } -bool CreateRelationNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateRelationNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_create_access(tdbb, SCL_object_table); - return true; } void CreateRelationNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -7542,12 +7516,11 @@ string AlterRelationNode::internalPrint(NodePrinter& printer) const return "AlterRelationNode"; } -bool AlterRelationNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void AlterRelationNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { dsc dscName; dscName.makeText(name.length(), CS_METADATA, (UCHAR*) name.c_str()); SCL_check_relation(tdbb, &dscName, SCL_alter); - return true; } void AlterRelationNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -8363,7 +8336,7 @@ string DropRelationNode::internalPrint(NodePrinter& printer) const return "DropRelationNode"; } -bool DropRelationNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropRelationNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { dsc dscName; dscName.makeText(name.length(), CS_METADATA, (UCHAR*) name.c_str()); @@ -8371,7 +8344,6 @@ bool DropRelationNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) SCL_check_view(tdbb, &dscName, SCL_drop); else SCL_check_relation(tdbb, &dscName, SCL_drop); - return true; } void DropRelationNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -8617,18 +8589,17 @@ DdlNode* CreateAlterViewNode::dsqlPass(DsqlCompilerScratch* dsqlScratch) return DdlNode::dsqlPass(dsqlScratch); } -bool CreateAlterViewNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateAlterViewNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { dsc dscName; dscName.makeText(name.length(), CS_METADATA, (UCHAR*) name.c_str()); if (alter) { if (SCL_check_view(tdbb, &dscName, SCL_alter) || !create) - return true; + return; } SCL_check_create_access(tdbb, SCL_object_view); - return true; } void CreateAlterViewNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, @@ -9765,13 +9736,12 @@ string CreateIndexNode::internalPrint(NodePrinter& printer) const return "CreateIndexNode"; } -bool CreateIndexNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateIndexNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { dsc dscName; const MetaName &relationName = relation->dsqlName; dscName.makeText(relationName.length(), CS_METADATA, (UCHAR*) relationName.c_str()); SCL_check_relation(tdbb, &dscName, SCL_alter, false); - return true; } // Define an index. @@ -9836,7 +9806,7 @@ string AlterIndexNode::internalPrint(NodePrinter& printer) const return "AlterIndexNode"; } -bool AlterIndexNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void AlterIndexNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { bool systemIndex; MetaName relationName = getIndexRelationName(tdbb, transaction, name, systemIndex); @@ -9845,7 +9815,6 @@ bool AlterIndexNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) dscName.makeText(relationName.length(), CS_METADATA, (UCHAR*) relationName.c_str()); SCL_check_relation(tdbb, &dscName, SCL_alter, systemIndex); - return true; } void AlterIndexNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction) @@ -9899,7 +9868,7 @@ string SetStatisticsNode::internalPrint(NodePrinter& printer) const return "SetStatisticsNode"; } -bool SetStatisticsNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void SetStatisticsNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { bool systemIndex; MetaName relationName = getIndexRelationName(tdbb, transaction, name, systemIndex); @@ -9908,7 +9877,6 @@ bool SetStatisticsNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) dscName.makeText(relationName.length(), CS_METADATA, (UCHAR*) relationName.c_str()); SCL_check_relation(tdbb, &dscName, SCL_alter, false); - return true; } void SetStatisticsNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction) @@ -9981,7 +9949,7 @@ string DropIndexNode::internalPrint(NodePrinter& printer) const return "DropIndexNode"; } -bool DropIndexNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropIndexNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { bool systemIndex; MetaName relationName = getIndexRelationName(tdbb, transaction, name, systemIndex); @@ -9990,7 +9958,6 @@ bool DropIndexNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) dscName.makeText(relationName.length(), CS_METADATA, (UCHAR*) relationName.c_str()); SCL_check_relation(tdbb, &dscName, SCL_alter, systemIndex); - return true; } void DropIndexNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction) @@ -10051,10 +10018,9 @@ string CreateFilterNode::internalPrint(NodePrinter& printer) const return "CreateFilterNode"; } -bool CreateFilterNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateFilterNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_create_access(tdbb, SCL_object_filter); - return true; } // Define a blob filter. @@ -10132,10 +10098,9 @@ string DropFilterNode::internalPrint(NodePrinter& printer) const return "DropFilterNode"; } -bool DropFilterNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropFilterNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_filter(tdbb, name, SCL_drop); - return true; } void DropFilterNode::execute(thread_db* tdbb, DsqlCompilerScratch* /*dsqlScratch*/, jrd_tra* transaction) @@ -10181,10 +10146,9 @@ string CreateShadowNode::internalPrint(NodePrinter& printer) const return "CreateShadowNode"; } -bool CreateShadowNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateShadowNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_database(tdbb, SCL_alter); - return true; } void CreateShadowNode::execute(thread_db* tdbb, DsqlCompilerScratch* /*dsqlScratch*/, jrd_tra* transaction) @@ -10251,10 +10215,9 @@ string DropShadowNode::internalPrint(NodePrinter& printer) const return "DropShadowNode"; } -bool DropShadowNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropShadowNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_database(tdbb, SCL_alter); - return true; } void DropShadowNode::execute(thread_db* tdbb, DsqlCompilerScratch* /*dsqlScratch*/, jrd_tra* transaction) @@ -10309,14 +10272,12 @@ string CreateAlterRoleNode::internalPrint(NodePrinter& printer) const return "CreateAlterRoleNode"; } -bool CreateAlterRoleNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateAlterRoleNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { if (createFlag) SCL_check_create_access(tdbb, SCL_object_role); else SCL_check_role(tdbb, name, SCL_alter); - - return true; } void CreateAlterRoleNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction) @@ -10493,10 +10454,9 @@ void MappingNode::addItem(string& ddl, const char* text, char quote) ddl += quote; } -bool MappingNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void MappingNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { //DDL_TODO - return false; } void MappingNode::runInSecurityDb(SecDbContext* secDbContext) @@ -10860,10 +10820,9 @@ string DropRoleNode::internalPrint(NodePrinter& printer) const return "DropRoleNode"; } -bool DropRoleNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropRoleNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_role(tdbb, name, SCL_drop); - return true; } void DropRoleNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction) @@ -10993,10 +10952,9 @@ static void setCharField(Auth::CharField& field, const string* value) } -bool CreateAlterUserNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateAlterUserNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { //DDL_TODO - return false; } @@ -11128,10 +11086,9 @@ string DropUserNode::internalPrint(NodePrinter& printer) const return "DropUserNode"; } -bool DropUserNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropUserNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { //DDL_TODO - return false; } void DropUserNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction) @@ -11185,10 +11142,9 @@ string GrantRevokeNode::internalPrint(NodePrinter& printer) const return "GrantRevokeNode"; } -bool GrantRevokeNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void GrantRevokeNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { // GRANT OPTION will be checked in grantRevoke method - return false; } void GrantRevokeNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction) @@ -12423,10 +12379,9 @@ string AlterDatabaseNode::internalPrint(NodePrinter& printer) const return "AlterDatabaseNode"; } -bool AlterDatabaseNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void AlterDatabaseNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_database(tdbb, SCL_alter); - return true; } void AlterDatabaseNode::checkClauses(thread_db* tdbb) diff --git a/src/dsql/DdlNodes.h b/src/dsql/DdlNodes.h index 919ee5e302..0cb61ed1d4 100644 --- a/src/dsql/DdlNodes.h +++ b/src/dsql/DdlNodes.h @@ -232,9 +232,10 @@ public: return "RecreateNode"; } - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction) + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction) { - return dropNode.checkPermission(tdbb, transaction) && createNode->checkPermission(tdbb, transaction); + dropNode.checkPermission(tdbb, transaction); + createNode->checkPermission(tdbb, transaction); } virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction) @@ -280,7 +281,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -308,7 +309,7 @@ public: } public: - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual Firebird::string internalPrint(NodePrinter& printer) const; virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); @@ -338,7 +339,7 @@ public: } public: - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual Firebird::string internalPrint(NodePrinter& printer) const; virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); @@ -371,7 +372,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -421,7 +422,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; virtual DdlNode* dsqlPass(DsqlCompilerScratch* dsqlScratch); - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -484,7 +485,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -517,7 +518,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; virtual DdlNode* dsqlPass(DsqlCompilerScratch* dsqlScratch); - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -563,7 +564,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; virtual DdlNode* dsqlPass(DsqlCompilerScratch* dsqlScratch); - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -624,7 +625,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; virtual DdlNode* dsqlPass(DsqlCompilerScratch* dsqlScratch); - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -717,7 +718,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; virtual DdlNode* dsqlPass(DsqlCompilerScratch* dsqlScratch); - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -791,7 +792,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; virtual DdlNode* dsqlPass(DsqlCompilerScratch* dsqlScratch); - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -831,7 +832,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; virtual DdlNode* dsqlPass(DsqlCompilerScratch* dsqlScratch); - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); void setAttribute(USHORT attribute) @@ -890,7 +891,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -917,7 +918,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -956,7 +957,7 @@ public: const Firebird::MetaName& newFieldName); virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -994,7 +995,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1026,7 +1027,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1063,7 +1064,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1113,7 +1114,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); virtual DdlNode* dsqlPass(DsqlCompilerScratch* dsqlScratch) @@ -1156,7 +1157,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1561,7 +1562,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1591,7 +1592,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1622,7 +1623,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1660,7 +1661,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; virtual DdlNode* dsqlPass(DsqlCompilerScratch* dsqlScratch); - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1742,7 +1743,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1773,7 +1774,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1799,7 +1800,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1828,7 +1829,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1887,7 +1888,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1916,7 +1917,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -1944,7 +1945,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); virtual bool mustBeReplicated() const @@ -1979,7 +1980,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); virtual bool mustBeReplicated() const @@ -2026,7 +2027,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -2079,7 +2080,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -2120,7 +2121,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -2166,7 +2167,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); virtual bool mustBeReplicated() const @@ -2234,7 +2235,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); virtual bool mustBeReplicated() const @@ -2292,7 +2293,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -2409,7 +2410,7 @@ public: } virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); virtual bool mustBeReplicated() const diff --git a/src/dsql/Nodes.h b/src/dsql/Nodes.h index edb7c5bc0a..f98d5e982c 100644 --- a/src/dsql/Nodes.h +++ b/src/dsql/Nodes.h @@ -190,7 +190,7 @@ public: // Check permission on DDL operation. Return true if everything is OK. // Raise an exception for bad permission. // If returns false permissions will be check in old style at vio level as well as while direct RDB$ tables modify. - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction) = 0; + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction) = 0; // Set the scratch's transaction when executing a node. Fact of accessing the scratch during // execution is a hack. diff --git a/src/dsql/PackageNodes.epp b/src/dsql/PackageNodes.epp index 23f5224451..2a33291f4d 100644 --- a/src/dsql/PackageNodes.epp +++ b/src/dsql/PackageNodes.epp @@ -279,7 +279,7 @@ DdlNode* CreateAlterPackageNode::dsqlPass(DsqlCompilerScratch* dsqlScratch) } -bool CreateAlterPackageNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreateAlterPackageNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { dsc dscName; dscName.makeText(name.length(), CS_METADATA, (UCHAR*) name.c_str()); @@ -287,11 +287,10 @@ bool CreateAlterPackageNode::checkPermission(thread_db* tdbb, jrd_tra* transacti if (alter) { if (SCL_check_package(tdbb, &dscName, SCL_alter) || !create) - return true; + return; } SCL_check_create_access(tdbb, SCL_object_package); - return true; } @@ -491,12 +490,11 @@ string DropPackageNode::internalPrint(NodePrinter& printer) const } -bool DropPackageNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropPackageNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { dsc dscName; dscName.makeText(name.length(), CS_METADATA, (UCHAR*) name.c_str()); SCL_check_package(tdbb, &dscName, SCL_drop); - return true; } @@ -687,10 +685,9 @@ DdlNode* CreatePackageBodyNode::dsqlPass(DsqlCompilerScratch* dsqlScratch) } -bool CreatePackageBodyNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void CreatePackageBodyNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { SCL_check_create_access(tdbb, SCL_object_package); - return true; } @@ -876,12 +873,11 @@ string DropPackageBodyNode::internalPrint(NodePrinter& printer) const } -bool DropPackageBodyNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) +void DropPackageBodyNode::checkPermission(thread_db* tdbb, jrd_tra* transaction) { dsc dscName; dscName.makeText(name.length(), CS_METADATA, (UCHAR*) name.c_str()); SCL_check_package(tdbb, &dscName, SCL_drop); - return true; } diff --git a/src/dsql/PackageNodes.h b/src/dsql/PackageNodes.h index 18da5eb828..7f181da659 100644 --- a/src/dsql/PackageNodes.h +++ b/src/dsql/PackageNodes.h @@ -84,7 +84,7 @@ public: public: virtual DdlNode* dsqlPass(DsqlCompilerScratch* dsqlScratch); virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -129,7 +129,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -164,7 +164,7 @@ public: public: virtual DdlNode* dsqlPass(DsqlCompilerScratch* dsqlScratch); virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: @@ -196,7 +196,7 @@ public: public: virtual Firebird::string internalPrint(NodePrinter& printer) const; - virtual bool checkPermission(thread_db* tdbb, jrd_tra* transaction); + virtual void checkPermission(thread_db* tdbb, jrd_tra* transaction); virtual void execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch, jrd_tra* transaction); protected: diff --git a/src/include/gen/msgs.h b/src/include/gen/msgs.h index 86e4745b30..26e9282725 100644 --- a/src/include/gen/msgs.h +++ b/src/include/gen/msgs.h @@ -968,7 +968,7 @@ Data source : @4"}, /* eds_statement */ {335545261, "Can not convert @1 to @2"}, /* bind_convert */ {335545262, "cannot update old BLOB"}, /* cannot_update_old_blob */ {335545263, "cannot read from new BLOB"}, /* cannot_read_new_blob */ - {335545264, "There is no privilege CREATE @1"}, /* dyn_no_create_priv */ + {335545264, "No permission for CREATE @1 operation"}, /* dyn_no_create_priv */ {335740929, "data base file name (@1) already given"}, /* gfix_db_name */ {335740930, "invalid switch @1"}, /* gfix_invalid_sw */ {335740932, "incompatible switch combination"}, /* gfix_incmp_sw */ diff --git a/src/jrd/jrd.h b/src/jrd/jrd.h index 176caa68c4..71747f746c 100644 --- a/src/jrd/jrd.h +++ b/src/jrd/jrd.h @@ -485,10 +485,10 @@ const ULONG TDBB_use_db_page_space = 128; // use database (not temporary) page const ULONG TDBB_detaching = 256; // detach is in progress const ULONG TDBB_wait_cancel_disable = 512; // don't cancel current waiting operation const ULONG TDBB_cache_unwound = 1024; // page cache was unwound -const ULONG TDBB_reset_stack = 4096; // stack should be reset after stack overflow exception -const ULONG TDBB_dfw_cleanup = 8192; // DFW cleanup phase is active -const ULONG TDBB_repl_sql = 16384; // SQL statement is being replicated -const ULONG TDBB_replicator = 32768; // Replicator +const ULONG TDBB_reset_stack = 2048; // stack should be reset after stack overflow exception +const ULONG TDBB_dfw_cleanup = 4096; // DFW cleanup phase is active +const ULONG TDBB_repl_sql = 8192; // SQL statement is being replicated +const ULONG TDBB_replicator = 16384; // Replicator class thread_db : public Firebird::ThreadData { diff --git a/src/jrd/scl.epp b/src/jrd/scl.epp index 824ad832ad..626e1ad00b 100644 --- a/src/jrd/scl.epp +++ b/src/jrd/scl.epp @@ -295,8 +295,7 @@ void SCL_check_create_access(thread_db* tdbb, int type) if (!(obj_mask & SCL_create)) { const char* name = accTypeNumToStr(type); -// ERR_post(Arg::Gds(isc_dyn_no_create_priv) << name); - ERR_post(Arg::Gds(isc_dyn_no_priv)); + ERR_post(Arg::Gds(isc_dyn_no_create_priv) << name); } } diff --git a/src/msgs/messages2.sql b/src/msgs/messages2.sql index 680008c239..8158f38344 100644 --- a/src/msgs/messages2.sql +++ b/src/msgs/messages2.sql @@ -1051,7 +1051,7 @@ Data source : @4', NULL, NULL) ('bind_convert', NULL, 'Coercion.cpp', NULL, 0, 941, NULL, 'Can not convert @1 to @2', NULL, NULL); ('cannot_update_old_blob', 'BLB_put_segment', 'blb.cpp', NULL, 0, 942, NULL, 'cannot update old BLOB', NULL, NULL); ('cannot_read_new_blob', 'BLB_get_segment', 'blb.cpp', NULL, 0, 943, NULL, 'cannot read from new BLOB', NULL, NULL); -('dyn_no_create_priv', NULL, 'scl.epp', NULL, 0, 944, NULL, 'There is no privilege CREATE @1', NULL, NULL); +('dyn_no_create_priv', NULL, 'scl.epp', NULL, 0, 944, NULL, 'No permission for CREATE @1 operation', NULL, NULL); -- QLI (NULL, NULL, NULL, NULL, 1, 0, NULL, 'expected type', NULL, NULL); (NULL, NULL, NULL, NULL, 1, 1, NULL, 'bad block type', NULL, NULL); From 9552ce2376a391526fdb52d26b9d87357e798c78 Mon Sep 17 00:00:00 2001 From: Roman Simakov Date: Tue, 28 Jan 2020 17:19:06 +0300 Subject: [PATCH 246/274] Deleted unneeded call of SCL_check_access if an object wasn't found --- src/jrd/scl.epp | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/src/jrd/scl.epp b/src/jrd/scl.epp index 626e1ad00b..2a1467affa 100644 --- a/src/jrd/scl.epp +++ b/src/jrd/scl.epp @@ -460,7 +460,11 @@ bool SCL_check_exception(thread_db* tdbb, const MetaName& name, SecurityClass::f } END_FOR - SCL_check_access(tdbb, s_class, 0, name, mask, SCL_object_exception, false, name); + if (s_class) + { + SCL_check_access(tdbb, s_class, 0, name, mask, SCL_object_exception, false, name); + return true; + } return found; } @@ -494,7 +498,11 @@ bool SCL_check_generator(thread_db* tdbb, const MetaName& name, SecurityClass::f } END_FOR - SCL_check_access(tdbb, s_class, 0, name, mask, SCL_object_generator, false, name); + if (s_class) + { + SCL_check_access(tdbb, s_class, 0, name, mask, SCL_object_generator, false, name); + return true; + } return found; } @@ -654,7 +662,11 @@ bool SCL_check_package(thread_db* tdbb, const dsc* dsc_name, SecurityClass::flag } END_FOR - SCL_check_access(tdbb, s_class, id_package, name, mask, SCL_object_package, false, name); + if (s_class) + { + SCL_check_access(tdbb, s_class, id_package, name, mask, SCL_object_package, false, name); + return true; + } return found; } @@ -697,7 +709,11 @@ bool SCL_check_procedure(thread_db* tdbb, const dsc* dsc_name, SecurityClass::fl } END_FOR - SCL_check_access(tdbb, s_class, id_procedure, name, mask, SCL_object_procedure, false, name); + if (s_class) + { + SCL_check_access(tdbb, s_class, id_procedure, name, mask, SCL_object_procedure, false, name); + return true; + } return found; } @@ -740,7 +756,11 @@ bool SCL_check_function(thread_db* tdbb, const dsc* dsc_name, SecurityClass::fla } END_FOR - SCL_check_access(tdbb, s_class, id_function, name, mask, SCL_object_function, false, name); + if (s_class) + { + SCL_check_access(tdbb, s_class, id_function, name, mask, SCL_object_function, false, name); + return true; + } return found; } @@ -855,14 +875,16 @@ bool SCL_check_view(thread_db* tdbb, const dsc* dsc_name, SecurityClass::flags_t WITH REL.RDB$RELATION_NAME EQ name.c_str() { if (!REL.RDB$SECURITY_CLASS.NULL) - { s_class = SCL_get_class(tdbb, REL.RDB$SECURITY_CLASS); - } found = true; } END_FOR - SCL_check_access(tdbb, s_class, 0, NULL, mask, SCL_object_view, false, name); + if (s_class) + { + SCL_check_access(tdbb, s_class, 0, NULL, mask, SCL_object_view, false, name); + return true; + } return found; } @@ -889,9 +911,7 @@ void SCL_check_role(thread_db* tdbb, const Firebird::MetaName& name, SecurityCla WITH R.RDB$ROLE_NAME EQ name.c_str() { if (!R.RDB$SECURITY_CLASS.NULL) - { s_class = SCL_get_class(tdbb, R.RDB$SECURITY_CLASS); - } } END_FOR From 0b43aa6985954df0f6d326f2d5593767f58f83b9 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 29 Jan 2020 00:05:05 +0000 Subject: [PATCH 247/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 46d566cf15..a1e98f0e82 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1743 + FORMAL BUILD NUMBER:1748 */ -#define PRODUCT_VER_STRING "4.0.0.1743" -#define FILE_VER_STRING "WI-T4.0.0.1743" -#define LICENSE_VER_STRING "WI-T4.0.0.1743" -#define FILE_VER_NUMBER 4, 0, 0, 1743 +#define PRODUCT_VER_STRING "4.0.0.1748" +#define FILE_VER_STRING "WI-T4.0.0.1748" +#define LICENSE_VER_STRING "WI-T4.0.0.1748" +#define FILE_VER_NUMBER 4, 0, 0, 1748 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1743" +#define FB_BUILD_NO "1748" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 53739c0de7..18b5bf8999 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1743 +BuildNum=1748 NowAt=`pwd` cd `dirname $0` From 61f656c411bced9dc459c278753c29e6642e63e2 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Tue, 28 Jan 2020 13:25:05 +0300 Subject: [PATCH 248/274] Use correct error codes when double precision value had been overflown --- src/common/DecFloat.cpp | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/common/DecFloat.cpp b/src/common/DecFloat.cpp index eca87138b1..ede40c1e89 100644 --- a/src/common/DecFloat.cpp +++ b/src/common/DecFloat.cpp @@ -58,28 +58,29 @@ struct Dec2fb { USHORT decError; ISC_STATUS fbError; + ISC_STATUS fbDoubleError; }; Dec2fb dec2fb[] = { - { DEC_IEEE_754_Division_by_zero, isc_decfloat_divide_by_zero }, - { DEC_IEEE_754_Inexact, isc_decfloat_inexact_result }, - { DEC_IEEE_754_Invalid_operation, isc_decfloat_invalid_operation }, - { DEC_IEEE_754_Overflow, isc_decfloat_overflow }, - { DEC_IEEE_754_Underflow, isc_decfloat_underflow }, + { DEC_IEEE_754_Division_by_zero, isc_decfloat_divide_by_zero, isc_exception_float_divide_by_zero }, + { DEC_IEEE_754_Inexact, isc_decfloat_inexact_result, isc_exception_float_inexact_result }, + { DEC_IEEE_754_Invalid_operation, isc_decfloat_invalid_operation, isc_exception_float_invalid_operand }, + { DEC_IEEE_754_Overflow, isc_decfloat_overflow, isc_exception_float_overflow }, + { DEC_IEEE_754_Underflow, isc_decfloat_underflow, isc_exception_float_underflow }, { 0, 0 } }; class DecimalContext : public decContext { public: - DecimalContext(const Decimal64*, DecimalStatus ds) - : decSt(ds) + DecimalContext(const Decimal64*, DecimalStatus ds, bool dblErr = false) + : decSt(ds), dblError(dblErr) { init(DEC_INIT_DECIMAL64); } - DecimalContext(const Decimal128*, DecimalStatus ds) - : decSt(ds) + DecimalContext(const Decimal128*, DecimalStatus ds, bool dblErr = false) + : decSt(ds), dblError(dblErr) { init(DEC_INIT_DECIMAL128); } @@ -106,12 +107,13 @@ public: { // Arg::Gds(isc_arith_except) as first vector element ? if (e->decError & unmaskedExceptions) - Arg::Gds(e->fbError).raise(); + Arg::Gds(dblError ? e->fbDoubleError : e->fbError).raise(); } } private: DecimalStatus decSt; + bool dblError; // Raise double proecision related errors instead decfloat void init(int kind) { @@ -622,7 +624,7 @@ void Decimal128::toString(string& to) const double Decimal128::toDouble(DecimalStatus decSt) const { - DecimalContext context(this, decSt); + DecimalContext context(this, decSt, true); if (compare(decSt, dmin) < 0) { @@ -656,7 +658,7 @@ SINT64 Decimal128::toInt64(DecimalStatus decSt, int scale) const if (wrk.compare(decSt, i64min) < 0 || wrk.compare(decSt, i64max) > 0) { - DecimalContext context(this, decSt); + DecimalContext context(this, decSt, true); decContextSetStatus(&context, DEC_Invalid_operation); return 0; // in case of no trap on invalid operation } From 60e628b123b4549818f576e8cb5234713ba3a784 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Wed, 29 Jan 2020 20:18:57 +0300 Subject: [PATCH 249/274] Make self test of compatibility UDR work with dialect 1 --- src/extlib/UdfBackwardCompatibility.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/extlib/UdfBackwardCompatibility.sql b/src/extlib/UdfBackwardCompatibility.sql index 8131e385eb..37cfc27a53 100644 --- a/src/extlib/UdfBackwardCompatibility.sql +++ b/src/extlib/UdfBackwardCompatibility.sql @@ -38,6 +38,9 @@ create function isLeapYear ( -- Run minimum test select 25, 3, div(25, 3) from rdb$database; select pi(), frac(pi()) from rdb$database; -select current_date, dow(current_date), sdow(current_date) from rdb$database; -select current_timestamp, getExactTimestampUTC() from rdb$database; -select current_date, isLeapYear(current_date) from rdb$database; +select timestamp '2020-01-29', dow(timestamp '2020-01-29'), sdow(timestamp '2020-01-29') from rdb$database; +set time zone 'utc'; +select cast(((current_timestamp - getexacttimestamputc()) * 1000) as integer) as getexacttimestamptest from rdb$database; +set time zone local; +select timestamp '2019-01-29', isleapyear(timestamp '2019-01-29') from rdb$database; +select timestamp '2020-01-29', isleapyear(timestamp '2020-01-29') from rdb$database; From 2ca63cce2ba63616612b14b55df1f4da819679b4 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Thu, 30 Jan 2020 00:05:06 +0000 Subject: [PATCH 250/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index a1e98f0e82..53ff430891 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1748 + FORMAL BUILD NUMBER:1750 */ -#define PRODUCT_VER_STRING "4.0.0.1748" -#define FILE_VER_STRING "WI-T4.0.0.1748" -#define LICENSE_VER_STRING "WI-T4.0.0.1748" -#define FILE_VER_NUMBER 4, 0, 0, 1748 +#define PRODUCT_VER_STRING "4.0.0.1750" +#define FILE_VER_STRING "WI-T4.0.0.1750" +#define LICENSE_VER_STRING "WI-T4.0.0.1750" +#define FILE_VER_NUMBER 4, 0, 0, 1750 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1748" +#define FB_BUILD_NO "1750" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 18b5bf8999..fb50141876 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1748 +BuildNum=1750 NowAt=`pwd` cd `dirname $0` From 28e3e2282aa514c00b3e9b1b834c04ba3dd3bee1 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Thu, 30 Jan 2020 13:07:24 +0300 Subject: [PATCH 251/274] Miss diagnostics --- src/burp/backup.epp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/burp/backup.epp b/src/burp/backup.epp index 68a4a233c9..8455f2baa0 100644 --- a/src/burp/backup.epp +++ b/src/burp/backup.epp @@ -403,8 +403,6 @@ int BACKUP_backup(const TEXT* dbb_file, const TEXT* file_name) write_mapping(); // Write database creators - BURP_verbose(391); - // msg 391 writing database creators write_db_creators(); } @@ -4131,6 +4129,7 @@ void write_db_creators() **************************************/ Firebird::IRequest* req_handle = nullptr; TEXT temp[GDS_NAME_LEN]; + bool first = true; BurpGlobals* tdgbl = BurpGlobals::getSpecific(); @@ -4139,6 +4138,13 @@ void write_db_creators() FOR (REQUEST_HANDLE req_handle) C IN RDB$DB_CREATORS + if (first) + { + BURP_verbose(391); + // msg 391 writing database create grants + first = false; + } + bool fl = true; if (!C.RDB$USER_TYPE.NULL) From 9e6c6d0066d3d1db051cad3bebc3847296751ce9 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Thu, 30 Jan 2020 17:22:56 +0300 Subject: [PATCH 252/274] Fixed CORE-6238: DECFLOAT: subtraction Num1 - Num2 leads to "Decimal float overflow" if Num2 is specified in scientific notation and less than max double ( 1.7976931348623157e308 ) --- src/common/dsc.cpp | 52 +++++++++++++++++++++--------------------- src/common/dsc.h | 2 +- src/dsql/ExprNodes.cpp | 40 ++++++++++++++++++++++++-------- 3 files changed, 58 insertions(+), 36 deletions(-) diff --git a/src/common/dsc.cpp b/src/common/dsc.cpp index 6601a26b36..5ea69093a2 100644 --- a/src/common/dsc.cpp +++ b/src/common/dsc.cpp @@ -203,7 +203,7 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_int64, dtype_int64, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, dtype_sql_date, dtype_sql_time, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec64, dtype_dec128, dtype_int128, dtype_sql_time_tz, dtype_timestamp_tz}, // dtype_long @@ -212,7 +212,7 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_int64, dtype_int64, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, dtype_sql_date, dtype_sql_time, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec64, dtype_dec128, dtype_int128, dtype_sql_time_tz, dtype_timestamp_tz}, // dtype_quad @@ -230,7 +230,7 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_double, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, dtype_sql_date, dtype_sql_time, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, dtype_double, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec64, dtype_dec128, dtype_double, dtype_sql_time_tz, dtype_timestamp_tz}, // dtype_double @@ -239,7 +239,7 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_double, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, dtype_sql_date, dtype_sql_time, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, dtype_double, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec64, dtype_dec128, dtype_double, dtype_sql_time_tz, dtype_timestamp_tz}, // dtype_d_float -- VMS deprecated @@ -248,7 +248,7 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_d_float, dtype_d_float, DTYPE_CANNOT, dtype_d_float, dtype_d_float, dtype_d_float, dtype_sql_date, dtype_sql_time, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_d_float, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_d_float, dtype_d_float, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec64, dtype_dec128, dtype_d_float, dtype_sql_time_tz, dtype_timestamp_tz}, // dtype_sql_date @@ -326,17 +326,17 @@ const BYTE DSC_add_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = // dtype_dec64 {dtype_unknown, dtype_dec128, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, - dtype_dec128, dtype_dec128, DTYPE_CANNOT, dtype_double, - dtype_double, dtype_d_float, dtype_sql_date, dtype_sql_time, + dtype_dec64, dtype_dec64, DTYPE_CANNOT, dtype_dec64, + dtype_dec64, dtype_dec64, dtype_sql_date, dtype_sql_time, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec64, dtype_dec128, dtype_dec128, dtype_sql_time_tz, dtype_timestamp_tz}, // dtype_dec128 {dtype_unknown, dtype_dec128, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, - dtype_dec128, dtype_dec128, DTYPE_CANNOT, dtype_double, - dtype_double, dtype_d_float, dtype_sql_date, dtype_sql_time, + dtype_dec128, dtype_dec128, DTYPE_CANNOT, dtype_dec128, + dtype_dec128, dtype_dec128, dtype_sql_date, dtype_sql_time, dtype_timestamp, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, dtype_dec128, dtype_sql_time_tz, dtype_timestamp_tz}, @@ -492,7 +492,7 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_double, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, dtype_double, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec64, dtype_dec128, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_double @@ -501,7 +501,7 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_double, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, dtype_double, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec64, dtype_dec128, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_d_float -- VMS deprecated @@ -510,7 +510,7 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_d_float, dtype_d_float, DTYPE_CANNOT, dtype_d_float, dtype_d_float, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_d_float, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_d_float, dtype_d_float, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec64, dtype_dec128, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_sql_date @@ -564,7 +564,7 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_int64, dtype_int64, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_int64, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec64, dtype_dec128, dtype_int128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_dbkey @@ -588,17 +588,17 @@ const BYTE DSC_sub_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = // dtype_dec64 {dtype_unknown, dtype_dec128, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, - dtype_dec128, dtype_dec128, DTYPE_CANNOT, dtype_double, - dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, + dtype_dec64, dtype_dec64, DTYPE_CANNOT, dtype_dec64, + dtype_dec64, dtype_dec64, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec64, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_dec128 {dtype_unknown, dtype_dec128, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, - dtype_dec128, dtype_dec128, DTYPE_CANNOT, dtype_double, - dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, + dtype_dec128, dtype_dec128, DTYPE_CANNOT, dtype_dec128, + dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT}, @@ -755,7 +755,7 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_double, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, dtype_double, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_double @@ -764,7 +764,7 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_double, dtype_double, DTYPE_CANNOT, dtype_double, dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_double, dtype_double, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, dtype_double, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_d_float -- VMS deprecated @@ -773,7 +773,7 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = dtype_d_float, dtype_d_float, DTYPE_CANNOT, dtype_d_float, dtype_d_float, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_d_float, - DTYPE_CANNOT, DTYPE_CANNOT, dtype_d_float, dtype_d_float, + DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT}, // dtype_sql_date @@ -851,8 +851,8 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = // dtype_dec64 {dtype_unknown, dtype_dec128, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, - dtype_dec128, dtype_dec128, DTYPE_CANNOT, dtype_double, - dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, + dtype_dec128, dtype_dec128, DTYPE_CANNOT, dtype_dec128, + dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT}, @@ -860,8 +860,8 @@ const BYTE DSC_multiply_result[DTYPE_TYPE_MAX][DTYPE_TYPE_MAX] = // dtype_dec128 {dtype_unknown, dtype_dec128, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, - dtype_dec128, dtype_dec128, DTYPE_CANNOT, dtype_double, - dtype_double, dtype_d_float, DTYPE_CANNOT, DTYPE_CANNOT, + dtype_dec128, dtype_dec128, DTYPE_CANNOT, dtype_dec128, + dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT, dtype_dec128, dtype_dec128, dtype_dec128, DTYPE_CANNOT, DTYPE_CANNOT}, diff --git a/src/common/dsc.h b/src/common/dsc.h index a29c6a5dda..e0f0f273e9 100644 --- a/src/common/dsc.h +++ b/src/common/dsc.h @@ -71,7 +71,7 @@ inline bool DTYPE_IS_APPROX(UCHAR d) inline bool DTYPE_IS_DECFLOAT(UCHAR d) { - return d == dtype_dec128 || d == dtype_dec64; + return d == dtype_dec128 || d == dtype_dec64; } inline bool DTYPE_IS_NUMERIC(UCHAR d) diff --git a/src/dsql/ExprNodes.cpp b/src/dsql/ExprNodes.cpp index decd7fc757..de84947fe7 100644 --- a/src/dsql/ExprNodes.cpp +++ b/src/dsql/ExprNodes.cpp @@ -896,12 +896,19 @@ void ArithmeticNode::makeDialect3(dsc* desc, dsc& desc1, dsc& desc2) else dtype = dtype_int64; } - else if (desc1.isDecOrInt() && desc2.isDecOrInt()) - dtype = dtype_dec128; else if (DTYPE_IS_NUMERIC(dtype1) && DTYPE_IS_NUMERIC(dtype2)) { - fb_assert(DTYPE_IS_APPROX(dtype1) || DTYPE_IS_APPROX(dtype2)); - dtype = dtype_double; + if (DTYPE_IS_DECFLOAT(dtype1) || DTYPE_IS_DECFLOAT(dtype2)) + { + USHORT d1 = DTYPE_IS_DECFLOAT(dtype1) ? dtype1 : 0; + USHORT d2 = DTYPE_IS_DECFLOAT(dtype2) ? dtype2 : 0; + dtype = MAX(d1, d2); + } + else + { + fb_assert(DTYPE_IS_APPROX(dtype1) || DTYPE_IS_APPROX(dtype2)); + dtype = dtype_double; + } } else { @@ -1478,10 +1485,20 @@ void ArithmeticNode::getDescDialect3(thread_db* /*tdbb*/, dsc* desc, dsc& desc1, else dtype = dtype_int64; } - else if (desc1.isDecOrInt() && desc2.isDecOrInt()) - dtype = dtype_dec128; - else if (DTYPE_IS_NUMERIC(desc1.dsc_dtype) && DTYPE_IS_NUMERIC(desc2.dsc_dtype)) - dtype = dtype_double; + else if (DTYPE_IS_NUMERIC(dtype1) && DTYPE_IS_NUMERIC(dtype2)) + { + if (DTYPE_IS_DECFLOAT(dtype1) || DTYPE_IS_DECFLOAT(dtype2)) + { + USHORT d1 = DTYPE_IS_DECFLOAT(dtype1) ? dtype1 : 0; + USHORT d2 = DTYPE_IS_DECFLOAT(dtype2) ? dtype2 : 0; + dtype = MAX(d1, d2); + } + else + { + fb_assert(DTYPE_IS_APPROX(dtype1) || DTYPE_IS_APPROX(dtype2)); + dtype = dtype_double; + } + } else { // mixed numeric and non-numeric: @@ -1498,7 +1515,12 @@ void ArithmeticNode::getDescDialect3(thread_db* /*tdbb*/, dsc* desc, dsc& desc1, dtype = MAX(dtype1, dtype2); } - +/* + if (dtype1 >= DTYPE_TYPE_MAX || dtype2 >= DTYPE_TYPE_MAX) + dtype = DTYPE_CANNOT; + else + dtype = (blrOp == blr_add) ? DSC_add_result[dtype1][dtype2] : DSC_sub_result[dtype1][dtype2]; +*/ switch (dtype) { case dtype_timestamp: From c9650051b929b8d238c69da2839a25c9c1b49423 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Thu, 30 Jan 2020 17:39:51 +0300 Subject: [PATCH 253/274] Comment --- src/remote/remote.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/remote/remote.h b/src/remote/remote.h index 8ffa2f8ced..241d820467 100644 --- a/src/remote/remote.h +++ b/src/remote/remote.h @@ -847,7 +847,7 @@ private: Firebird::ICryptKeyCallback* create(const Config* conf); - // Firebird::IClientBlock implementation + // Firebird::ICryptKeyCallback implementation unsigned callback(unsigned dataLength, const void* data, unsigned bufferLength, void* buffer); private: From fad521155eeb53489eff2b17427778c82d6cf00b Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Fri, 31 Jan 2020 00:04:36 +0000 Subject: [PATCH 254/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 53ff430891..e505185352 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1750 + FORMAL BUILD NUMBER:1753 */ -#define PRODUCT_VER_STRING "4.0.0.1750" -#define FILE_VER_STRING "WI-T4.0.0.1750" -#define LICENSE_VER_STRING "WI-T4.0.0.1750" -#define FILE_VER_NUMBER 4, 0, 0, 1750 +#define PRODUCT_VER_STRING "4.0.0.1753" +#define FILE_VER_STRING "WI-T4.0.0.1753" +#define LICENSE_VER_STRING "WI-T4.0.0.1753" +#define FILE_VER_NUMBER 4, 0, 0, 1753 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1750" +#define FB_BUILD_NO "1753" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index fb50141876..4c478ceb39 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1750 +BuildNum=1753 NowAt=`pwd` cd `dirname $0` From c2d8f87cada6ce7dbfec7af14a82f6cca18ca906 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 31 Jan 2020 12:47:09 +0300 Subject: [PATCH 255/274] Avoid code duplication --- src/jrd/scl.epp | 63 +++++++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/src/jrd/scl.epp b/src/jrd/scl.epp index 2a1467affa..17301b13df 100644 --- a/src/jrd/scl.epp +++ b/src/jrd/scl.epp @@ -77,6 +77,8 @@ static SecurityClass::flags_t walk_acl(thread_db* tdbb, const Acl&, const MetaNa SLONG, const Firebird::MetaName&); static void raiseError(thread_db* tdbb, SecurityClass::flags_t mask, SLONG type, const Firebird::MetaName& name, const Firebird::MetaName& r_name, const MetaName &invoker); +static bool check_object(thread_db* tdbb, bool found, const SecurityClass* s_class, SLONG obj_type, + const Firebird::MetaName& obj_name, SecurityClass::flags_t mask, SLONG type, const Firebird::MetaName& name); namespace @@ -460,12 +462,7 @@ bool SCL_check_exception(thread_db* tdbb, const MetaName& name, SecurityClass::f } END_FOR - if (s_class) - { - SCL_check_access(tdbb, s_class, 0, name, mask, SCL_object_exception, false, name); - return true; - } - return found; + return check_object(tdbb, found, s_class, 0, name, mask, SCL_object_exception, name); } @@ -498,12 +495,7 @@ bool SCL_check_generator(thread_db* tdbb, const MetaName& name, SecurityClass::f } END_FOR - if (s_class) - { - SCL_check_access(tdbb, s_class, 0, name, mask, SCL_object_generator, false, name); - return true; - } - return found; + return check_object(tdbb, found, s_class, 0, name, mask, SCL_object_generator, name); } @@ -662,12 +654,7 @@ bool SCL_check_package(thread_db* tdbb, const dsc* dsc_name, SecurityClass::flag } END_FOR - if (s_class) - { - SCL_check_access(tdbb, s_class, id_package, name, mask, SCL_object_package, false, name); - return true; - } - return found; + return check_object(tdbb, found, s_class, id_package, name, mask, SCL_object_package, name); } @@ -709,12 +696,7 @@ bool SCL_check_procedure(thread_db* tdbb, const dsc* dsc_name, SecurityClass::fl } END_FOR - if (s_class) - { - SCL_check_access(tdbb, s_class, id_procedure, name, mask, SCL_object_procedure, false, name); - return true; - } - return found; + return check_object(tdbb, found, s_class, id_procedure, name, mask, SCL_object_procedure, name); } @@ -756,12 +738,7 @@ bool SCL_check_function(thread_db* tdbb, const dsc* dsc_name, SecurityClass::fla } END_FOR - if (s_class) - { - SCL_check_access(tdbb, s_class, id_function, name, mask, SCL_object_function, false, name); - return true; - } - return found; + return check_object(tdbb, found, s_class, id_function, name, mask, SCL_object_function, name); } @@ -880,12 +857,7 @@ bool SCL_check_view(thread_db* tdbb, const dsc* dsc_name, SecurityClass::flags_t } END_FOR - if (s_class) - { - SCL_check_access(tdbb, s_class, 0, NULL, mask, SCL_object_view, false, name); - return true; - } - return found; + return check_object(tdbb, found, s_class, 0, NULL, mask, SCL_object_view, name); } void SCL_check_role(thread_db* tdbb, const Firebird::MetaName& name, SecurityClass::flags_t mask) @@ -1847,3 +1819,22 @@ USHORT SCL_convert_privilege(thread_db* tdbb, jrd_tra* transaction, const Firebi return rc; } + +// check object existence and access rights +static bool check_object(thread_db* tdbb, + bool found, + const SecurityClass* s_class, + SLONG obj_type, + const Firebird::MetaName& obj_name, + SecurityClass::flags_t mask, + SLONG type, + const Firebird::MetaName& name) +{ + if (s_class) + { + SCL_check_access(tdbb, s_class, obj_type, obj_name, mask, type, false, name); + return true; + } + return found; +} + From 678c48a1fc7f1d0ddc6cf59bc149c453e6d6d7a8 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 31 Jan 2020 12:53:04 +0300 Subject: [PATCH 256/274] Fixed gcc warning --- src/remote/client/interface.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/remote/client/interface.cpp b/src/remote/client/interface.cpp index abb2ecbfda..6c60eef788 100644 --- a/src/remote/client/interface.cpp +++ b/src/remote/client/interface.cpp @@ -8743,8 +8743,8 @@ ClntAuthBlock::ClntAuthBlock(const Firebird::PathName* fileName, Firebird::Clump cliUserName(getPool()), cliPassword(getPool()), cliOrigUserName(getPool()), dataForPlugin(getPool()), dataFromPlugin(getPool()), cryptKeys(getPool()), dpbConfig(getPool()), dpbPlugins(getPool()), - plugins(IPluginManager::TYPE_AUTH_CLIENT), authComplete(false), firstTime(true), - createdInterface(nullptr) + createdInterface(nullptr), + plugins(IPluginManager::TYPE_AUTH_CLIENT), authComplete(false), firstTime(true) { if (dpb && tags) { From 4436a189f646d4476f693c4944b49f4a1b729345 Mon Sep 17 00:00:00 2001 From: hvlad Date: Fri, 31 Jan 2020 14:32:56 +0200 Subject: [PATCH 257/274] Update Windows build --- builds/win32/msvc10/fbserver.vcxproj | 1 + builds/win32/msvc10/fbserver.vcxproj.filters | 3 +++ builds/win32/msvc10/legacy_auth.vcxproj | 1 + builds/win32/msvc10/legacy_auth.vcxproj.filters | 3 +++ builds/win32/msvc12/fbserver.vcxproj | 1 + builds/win32/msvc12/fbserver.vcxproj.filters | 3 +++ builds/win32/msvc12/legacy_auth.vcxproj | 1 + builds/win32/msvc12/legacy_auth.vcxproj.filters | 3 +++ builds/win32/msvc14/fbserver.vcxproj | 1 + builds/win32/msvc14/fbserver.vcxproj.filters | 3 +++ builds/win32/msvc14/legacy_auth.vcxproj | 1 + builds/win32/msvc14/legacy_auth.vcxproj.filters | 3 +++ builds/win32/msvc15/fbserver.vcxproj | 1 + builds/win32/msvc15/fbserver.vcxproj.filters | 3 +++ builds/win32/msvc15/legacy_auth.vcxproj | 1 + builds/win32/msvc15/legacy_auth.vcxproj.filters | 3 +++ 16 files changed, 32 insertions(+) diff --git a/builds/win32/msvc10/fbserver.vcxproj b/builds/win32/msvc10/fbserver.vcxproj index af1c19aeb9..53b1dbe2b3 100644 --- a/builds/win32/msvc10/fbserver.vcxproj +++ b/builds/win32/msvc10/fbserver.vcxproj @@ -191,6 +191,7 @@ + diff --git a/builds/win32/msvc10/fbserver.vcxproj.filters b/builds/win32/msvc10/fbserver.vcxproj.filters index d1dcdd39d1..32da3823e4 100644 --- a/builds/win32/msvc10/fbserver.vcxproj.filters +++ b/builds/win32/msvc10/fbserver.vcxproj.filters @@ -34,6 +34,9 @@ AUTH files + + AUTH files + diff --git a/builds/win32/msvc10/legacy_auth.vcxproj b/builds/win32/msvc10/legacy_auth.vcxproj index 55012f9c96..4e6a6ed70b 100644 --- a/builds/win32/msvc10/legacy_auth.vcxproj +++ b/builds/win32/msvc10/legacy_auth.vcxproj @@ -184,6 +184,7 @@ + diff --git a/builds/win32/msvc10/legacy_auth.vcxproj.filters b/builds/win32/msvc10/legacy_auth.vcxproj.filters index c4cda46f02..2502900a3f 100644 --- a/builds/win32/msvc10/legacy_auth.vcxproj.filters +++ b/builds/win32/msvc10/legacy_auth.vcxproj.filters @@ -20,5 +20,8 @@ AUTH files + + AUTH files + \ No newline at end of file diff --git a/builds/win32/msvc12/fbserver.vcxproj b/builds/win32/msvc12/fbserver.vcxproj index 880b0a1056..908d46b426 100644 --- a/builds/win32/msvc12/fbserver.vcxproj +++ b/builds/win32/msvc12/fbserver.vcxproj @@ -195,6 +195,7 @@ + diff --git a/builds/win32/msvc12/fbserver.vcxproj.filters b/builds/win32/msvc12/fbserver.vcxproj.filters index a32b3cd75f..d20a985e5f 100644 --- a/builds/win32/msvc12/fbserver.vcxproj.filters +++ b/builds/win32/msvc12/fbserver.vcxproj.filters @@ -37,6 +37,9 @@ AUTH files + + AUTH files + Replication diff --git a/builds/win32/msvc12/legacy_auth.vcxproj b/builds/win32/msvc12/legacy_auth.vcxproj index 8d1299fcb6..0c9671cae4 100644 --- a/builds/win32/msvc12/legacy_auth.vcxproj +++ b/builds/win32/msvc12/legacy_auth.vcxproj @@ -188,6 +188,7 @@ + diff --git a/builds/win32/msvc12/legacy_auth.vcxproj.filters b/builds/win32/msvc12/legacy_auth.vcxproj.filters index c4cda46f02..2502900a3f 100644 --- a/builds/win32/msvc12/legacy_auth.vcxproj.filters +++ b/builds/win32/msvc12/legacy_auth.vcxproj.filters @@ -20,5 +20,8 @@ AUTH files + + AUTH files + \ No newline at end of file diff --git a/builds/win32/msvc14/fbserver.vcxproj b/builds/win32/msvc14/fbserver.vcxproj index 463065149c..ea6329646f 100644 --- a/builds/win32/msvc14/fbserver.vcxproj +++ b/builds/win32/msvc14/fbserver.vcxproj @@ -195,6 +195,7 @@ + diff --git a/builds/win32/msvc14/fbserver.vcxproj.filters b/builds/win32/msvc14/fbserver.vcxproj.filters index a32b3cd75f..d20a985e5f 100644 --- a/builds/win32/msvc14/fbserver.vcxproj.filters +++ b/builds/win32/msvc14/fbserver.vcxproj.filters @@ -37,6 +37,9 @@ AUTH files + + AUTH files + Replication diff --git a/builds/win32/msvc14/legacy_auth.vcxproj b/builds/win32/msvc14/legacy_auth.vcxproj index ed586ff60c..067b63ea36 100644 --- a/builds/win32/msvc14/legacy_auth.vcxproj +++ b/builds/win32/msvc14/legacy_auth.vcxproj @@ -188,6 +188,7 @@ + diff --git a/builds/win32/msvc14/legacy_auth.vcxproj.filters b/builds/win32/msvc14/legacy_auth.vcxproj.filters index c4cda46f02..2502900a3f 100644 --- a/builds/win32/msvc14/legacy_auth.vcxproj.filters +++ b/builds/win32/msvc14/legacy_auth.vcxproj.filters @@ -20,5 +20,8 @@ AUTH files + + AUTH files + \ No newline at end of file diff --git a/builds/win32/msvc15/fbserver.vcxproj b/builds/win32/msvc15/fbserver.vcxproj index f5cb4f97ff..0418db84c0 100644 --- a/builds/win32/msvc15/fbserver.vcxproj +++ b/builds/win32/msvc15/fbserver.vcxproj @@ -196,6 +196,7 @@ + diff --git a/builds/win32/msvc15/fbserver.vcxproj.filters b/builds/win32/msvc15/fbserver.vcxproj.filters index a32b3cd75f..d20a985e5f 100644 --- a/builds/win32/msvc15/fbserver.vcxproj.filters +++ b/builds/win32/msvc15/fbserver.vcxproj.filters @@ -37,6 +37,9 @@ AUTH files + + AUTH files + Replication diff --git a/builds/win32/msvc15/legacy_auth.vcxproj b/builds/win32/msvc15/legacy_auth.vcxproj index 85e6129630..66824c107f 100644 --- a/builds/win32/msvc15/legacy_auth.vcxproj +++ b/builds/win32/msvc15/legacy_auth.vcxproj @@ -189,6 +189,7 @@ + diff --git a/builds/win32/msvc15/legacy_auth.vcxproj.filters b/builds/win32/msvc15/legacy_auth.vcxproj.filters index c4cda46f02..2502900a3f 100644 --- a/builds/win32/msvc15/legacy_auth.vcxproj.filters +++ b/builds/win32/msvc15/legacy_auth.vcxproj.filters @@ -20,5 +20,8 @@ AUTH files + + AUTH files + \ No newline at end of file From 0ef5a1a1c1bf42021b378e1691aaccfd75a454b4 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 31 Jan 2020 20:20:02 +0300 Subject: [PATCH 258/274] Postfix for CORE-6238 --- src/dsql/ExprNodes.cpp | 9 ++------- src/jrd/cvt2.cpp | 17 +++++++++-------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/dsql/ExprNodes.cpp b/src/dsql/ExprNodes.cpp index de84947fe7..3355ada9fa 100644 --- a/src/dsql/ExprNodes.cpp +++ b/src/dsql/ExprNodes.cpp @@ -1513,14 +1513,9 @@ void ArithmeticNode::getDescDialect3(thread_db* /*tdbb*/, dsc* desc, dsc& desc1, if (dtype_int64 == dtype2) dtype2 = dtype_double; - dtype = MAX(dtype1, dtype2); + dtype = CVT2_compare_priority[dtype1] > CVT2_compare_priority[dtype2] ? dtype1 : dtype2; } -/* - if (dtype1 >= DTYPE_TYPE_MAX || dtype2 >= DTYPE_TYPE_MAX) - dtype = DTYPE_CANNOT; - else - dtype = (blrOp == blr_add) ? DSC_add_result[dtype1][dtype2] : DSC_sub_result[dtype1][dtype2]; -*/ + switch (dtype) { case dtype_timestamp: diff --git a/src/jrd/cvt2.cpp b/src/jrd/cvt2.cpp index 8959088287..eb8a77bb51 100644 --- a/src/jrd/cvt2.cpp +++ b/src/jrd/cvt2.cpp @@ -76,19 +76,20 @@ const BYTE CVT2_compare_priority[] = 1, // dtype_text 2, // dtype_cstring 3, // dtype_varying - // dtypes and 4, 5 are unused. + // dtypes and 4, 5 are unused 0, 0, // packed through long also have their natural values in the table 6, // dtype_packed 7, // dtype_byte, 8, // dtype_short 9, // dtype_long - // Move quad up by one to make room for int64 at its proper place in the table. + // Move quad up by one to make room for int64 at its proper place in the table 11, // dtype_quad - // Also leave space for int128, dec64 and dec 128. - 15, // dtype_real - 16, // dtype_double - 17, // dtype_d_float + // Leave space for int128 + 13, // dtype_real + 14, // dtype_double + 15, // dtype_d_float + // Leave space for dec64 and dec128 18, // dtype_sql_date 19, // dtype_sql_time // Leave space for dtype_sql_time_tz @@ -100,8 +101,8 @@ const BYTE CVT2_compare_priority[] = 25, // dtype_dbkey - compares with nothing except itself 26, // dtype_boolean - compares with nothing except itself 12, // dtype_int128 - go after quad - 13, // dec64 - go after dtype_int128 - 14, // dec128 - go after dec64 and before real + 16, // dec64 - go after dtype_d_float + 17, // dec128 - go after dec64 and before dtype_sql_date 20, // dtype_sql_time_tz - go after dtype_sql_time 22 // dtype_timestamp_tz - go after dtype_timestamp }; From f32cd4b403517c341d8c14078be9821fef4a644c Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 1 Feb 2020 00:04:42 +0000 Subject: [PATCH 259/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index e505185352..ca42baea5e 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1753 + FORMAL BUILD NUMBER:1759 */ -#define PRODUCT_VER_STRING "4.0.0.1753" -#define FILE_VER_STRING "WI-T4.0.0.1753" -#define LICENSE_VER_STRING "WI-T4.0.0.1753" -#define FILE_VER_NUMBER 4, 0, 0, 1753 +#define PRODUCT_VER_STRING "4.0.0.1759" +#define FILE_VER_STRING "WI-T4.0.0.1759" +#define LICENSE_VER_STRING "WI-T4.0.0.1759" +#define FILE_VER_NUMBER 4, 0, 0, 1759 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1753" +#define FB_BUILD_NO "1759" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 4c478ceb39..80623f8942 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1753 +BuildNum=1759 NowAt=`pwd` cd `dirname $0` From 272a49e3eaf1cfb1e69e15104d3fbb386c3777a2 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Mon, 3 Feb 2020 17:51:57 +0300 Subject: [PATCH 260/274] Postfix for CORE-5974 & CORE-6218 --- src/jrd/sort.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jrd/sort.cpp b/src/jrd/sort.cpp index f44bc99bd3..37f5f81640 100644 --- a/src/jrd/sort.cpp +++ b/src/jrd/sort.cpp @@ -825,7 +825,7 @@ void Sort::diddleKey(UCHAR* record, bool direction, bool duplicateHandling) ((Decimal64*) p)->makeKey(lwp); *p ^= 1 << 7; } - else if (!(key->skd_flags & SKD_separate_data)) + else if (duplicateHandling || !(key->skd_flags & SKD_separate_data)) { if (complement && n) { @@ -847,7 +847,7 @@ void Sort::diddleKey(UCHAR* record, bool direction, bool duplicateHandling) ((Decimal128*) p)->makeKey(lwp); *p ^= 1 << 7; } - else if (!(key->skd_flags & SKD_separate_data)) + else if (duplicateHandling || !(key->skd_flags & SKD_separate_data)) { if (complement && n) { @@ -1135,7 +1135,7 @@ void Sort::diddleKey(UCHAR* record, bool direction, bool duplicateHandling) ((Decimal64*) p)->makeKey(lwp); p[3] ^= 1 << 7; } - else if (!(key->skd_flags & SKD_separate_data)) + else if (duplicateHandling || !(key->skd_flags & SKD_separate_data)) { if (complement && n) { @@ -1156,7 +1156,7 @@ void Sort::diddleKey(UCHAR* record, bool direction, bool duplicateHandling) ((Decimal128*) p)->makeKey(lwp); p[3] ^= 1 << 7; } - else if (!(key->skd_flags & SKD_separate_data)) + else if (duplicateHandling || !(key->skd_flags & SKD_separate_data)) { if (complement && n) { From fcae494a2f7da91ebe81c0e68a18264809837509 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Mon, 3 Feb 2020 15:41:35 +0000 Subject: [PATCH 261/274] Fixed CORE-6241 - Values greater than number of days between 01.01.0001 and 31.12.9999 (=3652058) can be added or subtracted from DATE. --- src/dsql/ExprNodes.cpp | 13 +++++++++++++ src/jrd/SysFunction.cpp | 24 ++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/dsql/ExprNodes.cpp b/src/dsql/ExprNodes.cpp index 3355ada9fa..f81e00157e 100644 --- a/src/dsql/ExprNodes.cpp +++ b/src/dsql/ExprNodes.cpp @@ -2569,8 +2569,13 @@ dsc* ArithmeticNode::addSqlDate(const dsc* desc, impure_value* value) const op1_is_date = true; } else + { d1 = MOV_get_int64(tdbb, &value->vlu_desc, 0); + if (abs(d1) > TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) + ERR_post(Arg::Gds(isc_date_range_exceeded)); + } + SINT64 d2; // Coerce operand2 to a count of days bool op2_is_date = false; @@ -2580,8 +2585,13 @@ dsc* ArithmeticNode::addSqlDate(const dsc* desc, impure_value* value) const op2_is_date = true; } else + { d2 = MOV_get_int64(tdbb, desc, 0); + if (abs(d2) > TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) + ERR_post(Arg::Gds(isc_date_range_exceeded)); + } + if (blrOp == blr_subtract && op1_is_date && op2_is_date) { d2 = d1 - d2; @@ -13504,6 +13514,9 @@ static SINT64 getDayFraction(const dsc* d) // Convert the input number to a double CVT_move(d, &result, tdbb->getAttachment()->att_dec_status); + if (abs((SINT64) result_days) > TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) + ERR_post(Arg::Gds(isc_date_range_exceeded)); + // There's likely some loss of precision here due to rounding of number // 08-Apr-2004, Nickolay Samofatov. Loss of precision manifested itself as bad diff --git a/src/jrd/SysFunction.cpp b/src/jrd/SysFunction.cpp index be5e09b45d..d824b987b6 100644 --- a/src/jrd/SysFunction.cpp +++ b/src/jrd/SysFunction.cpp @@ -2332,10 +2332,11 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr switch (part) { - // TO DO: detect overflow in the following cases. - case blr_extract_year: { + if (abs(quantity) > 9999) + ERR_post(Arg::Gds(isc_date_range_exceeded)); + tm times; timestamp.decode(×); times.tm_year += quantity; @@ -2351,6 +2352,9 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr case blr_extract_month: { + if (abs(quantity) > 9999 * 12) + ERR_post(Arg::Gds(isc_date_range_exceeded)); + tm times; timestamp.decode(×); @@ -2392,14 +2396,21 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr break; case blr_extract_day: + if (abs(quantity) > TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) + ERR_post(Arg::Gds(isc_date_range_exceeded)); timestamp.value().timestamp_date += quantity; break; case blr_extract_week: + if (abs(quantity) > (TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) / 7 + 1) + ERR_post(Arg::Gds(isc_date_range_exceeded)); timestamp.value().timestamp_date += quantity * 7; break; case blr_extract_hour: + if (abs(quantity) > SINT64(TimeStamp::MAX_DATE - TimeStamp::MIN_DATE + 1) * 24) + ERR_post(Arg::Gds(isc_date_range_exceeded)); + if (valueDsc->dsc_dtype == dtype_sql_date) timestamp.value().timestamp_date += quantity / 24; else @@ -2407,6 +2418,9 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr break; case blr_extract_minute: + if (abs(quantity) > SINT64(TimeStamp::MAX_DATE - TimeStamp::MIN_DATE + 1) * 24 * 60) + ERR_post(Arg::Gds(isc_date_range_exceeded)); + if (valueDsc->dsc_dtype == dtype_sql_date) timestamp.value().timestamp_date += quantity / 1440; // 1440 == 24 * 60 else @@ -2414,6 +2428,9 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr break; case blr_extract_second: + if (abs(quantity) > SINT64(TimeStamp::MAX_DATE - TimeStamp::MIN_DATE + 1) * 24 * 60 * 60) + ERR_post(Arg::Gds(isc_date_range_exceeded)); + if (valueDsc->dsc_dtype == dtype_sql_date) timestamp.value().timestamp_date += quantity / oneDay; else @@ -2421,6 +2438,9 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr break; case blr_extract_millisecond: + if (abs(quantity) > SINT64(TimeStamp::MAX_DATE - TimeStamp::MIN_DATE + 1) * 24 * 60 * 60 * 1000) + ERR_post(Arg::Gds(isc_date_range_exceeded)); + if (valueDsc->dsc_dtype == dtype_sql_date) timestamp.value().timestamp_date += quantity / milliPow / (oneDay * 1000); else From 7b81b12390a35120b2d9f3a9acf718dd43cf5a10 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Tue, 4 Feb 2020 00:05:05 +0000 Subject: [PATCH 262/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index ca42baea5e..2b5d3574a4 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1759 + FORMAL BUILD NUMBER:1761 */ -#define PRODUCT_VER_STRING "4.0.0.1759" -#define FILE_VER_STRING "WI-T4.0.0.1759" -#define LICENSE_VER_STRING "WI-T4.0.0.1759" -#define FILE_VER_NUMBER 4, 0, 0, 1759 +#define PRODUCT_VER_STRING "4.0.0.1761" +#define FILE_VER_STRING "WI-T4.0.0.1761" +#define LICENSE_VER_STRING "WI-T4.0.0.1761" +#define FILE_VER_NUMBER 4, 0, 0, 1761 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1759" +#define FB_BUILD_NO "1761" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 80623f8942..3716ec5707 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1759 +BuildNum=1761 NowAt=`pwd` cd `dirname $0` From 53e74d2aa427c6d567397d8046dbdba97f6cc49a Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Mon, 3 Feb 2020 22:38:26 -0300 Subject: [PATCH 263/274] Postfix for CORE-6241. --- src/common/utils_proto.h | 9 +++++++++ src/dsql/ExprNodes.cpp | 6 +++--- src/jrd/SysFunction.cpp | 22 ++++++++++++++-------- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/common/utils_proto.h b/src/common/utils_proto.h index f49b7ff1bf..305e4d351a 100644 --- a/src/common/utils_proto.h +++ b/src/common/utils_proto.h @@ -104,6 +104,15 @@ namespace fb_utils bool isGlobalKernelPrefix(); #endif + // Compare the absolute value of two SINT64 numbers. + // Return 0 if they are equal, <0 if n1 < n2 and >0 if n1 > n2. + inline int abs64Compare(SINT64 n1, SINT64 n2) + { + n1 = n1 > 0 ? -n1 : n1; + n2 = n2 > 0 ? -n2 : n2; + return n1 == n2 ? 0 : n1 < n2 ? 1 : -1; + } + Firebird::PathName get_process_name(); SLONG genUniqueId(); void getCwd(Firebird::PathName& pn); diff --git a/src/dsql/ExprNodes.cpp b/src/dsql/ExprNodes.cpp index f81e00157e..52c651a38a 100644 --- a/src/dsql/ExprNodes.cpp +++ b/src/dsql/ExprNodes.cpp @@ -2572,7 +2572,7 @@ dsc* ArithmeticNode::addSqlDate(const dsc* desc, impure_value* value) const { d1 = MOV_get_int64(tdbb, &value->vlu_desc, 0); - if (abs(d1) > TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) + if (fb_utils::abs64Compare(d1, TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) > 0) ERR_post(Arg::Gds(isc_date_range_exceeded)); } @@ -2588,7 +2588,7 @@ dsc* ArithmeticNode::addSqlDate(const dsc* desc, impure_value* value) const { d2 = MOV_get_int64(tdbb, desc, 0); - if (abs(d2) > TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) + if (fb_utils::abs64Compare(d2, TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) > 0) ERR_post(Arg::Gds(isc_date_range_exceeded)); } @@ -13514,7 +13514,7 @@ static SINT64 getDayFraction(const dsc* d) // Convert the input number to a double CVT_move(d, &result, tdbb->getAttachment()->att_dec_status); - if (abs((SINT64) result_days) > TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) + if (fb_utils::abs64Compare(result_days, TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) > 0) ERR_post(Arg::Gds(isc_date_range_exceeded)); // There's likely some loss of precision here due to rounding of number diff --git a/src/jrd/SysFunction.cpp b/src/jrd/SysFunction.cpp index d824b987b6..6ae95e42aa 100644 --- a/src/jrd/SysFunction.cpp +++ b/src/jrd/SysFunction.cpp @@ -2334,7 +2334,7 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr { case blr_extract_year: { - if (abs(quantity) > 9999) + if (fb_utils::abs64Compare(quantity, 9999) > 0) ERR_post(Arg::Gds(isc_date_range_exceeded)); tm times; @@ -2352,7 +2352,7 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr case blr_extract_month: { - if (abs(quantity) > 9999 * 12) + if (fb_utils::abs64Compare(quantity, 9999 * 12) > 0) ERR_post(Arg::Gds(isc_date_range_exceeded)); tm times; @@ -2396,19 +2396,19 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr break; case blr_extract_day: - if (abs(quantity) > TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) + if (fb_utils::abs64Compare(quantity, TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) > 0) ERR_post(Arg::Gds(isc_date_range_exceeded)); timestamp.value().timestamp_date += quantity; break; case blr_extract_week: - if (abs(quantity) > (TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) / 7 + 1) + if (fb_utils::abs64Compare(quantity, (TimeStamp::MAX_DATE - TimeStamp::MIN_DATE) / 7 + 1) > 0) ERR_post(Arg::Gds(isc_date_range_exceeded)); timestamp.value().timestamp_date += quantity * 7; break; case blr_extract_hour: - if (abs(quantity) > SINT64(TimeStamp::MAX_DATE - TimeStamp::MIN_DATE + 1) * 24) + if (fb_utils::abs64Compare(quantity, SINT64(TimeStamp::MAX_DATE - TimeStamp::MIN_DATE + 1) * 24) > 0) ERR_post(Arg::Gds(isc_date_range_exceeded)); if (valueDsc->dsc_dtype == dtype_sql_date) @@ -2418,7 +2418,7 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr break; case blr_extract_minute: - if (abs(quantity) > SINT64(TimeStamp::MAX_DATE - TimeStamp::MIN_DATE + 1) * 24 * 60) + if (fb_utils::abs64Compare(quantity, SINT64(TimeStamp::MAX_DATE - TimeStamp::MIN_DATE + 1) * 24 * 60) > 0) ERR_post(Arg::Gds(isc_date_range_exceeded)); if (valueDsc->dsc_dtype == dtype_sql_date) @@ -2428,8 +2428,11 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr break; case blr_extract_second: - if (abs(quantity) > SINT64(TimeStamp::MAX_DATE - TimeStamp::MIN_DATE + 1) * 24 * 60 * 60) + if (fb_utils::abs64Compare(quantity, + SINT64(TimeStamp::MAX_DATE - TimeStamp::MIN_DATE + 1) * 24 * 60 * 60) > 0) + { ERR_post(Arg::Gds(isc_date_range_exceeded)); + } if (valueDsc->dsc_dtype == dtype_sql_date) timestamp.value().timestamp_date += quantity / oneDay; @@ -2438,8 +2441,11 @@ dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArr break; case blr_extract_millisecond: - if (abs(quantity) > SINT64(TimeStamp::MAX_DATE - TimeStamp::MIN_DATE + 1) * 24 * 60 * 60 * 1000) + if (fb_utils::abs64Compare(quantity, + SINT64(TimeStamp::MAX_DATE - TimeStamp::MIN_DATE + 1) * 24 * 60 * 60 * 1000) > 0) + { ERR_post(Arg::Gds(isc_date_range_exceeded)); + } if (valueDsc->dsc_dtype == dtype_sql_date) timestamp.value().timestamp_date += quantity / milliPow / (oneDay * 1000); From b2b5f9a87cea26a9f12fa231804dba9d0426d3fa Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Tue, 4 Feb 2020 15:42:02 +0000 Subject: [PATCH 264/274] Improvement CORE-6239 - Procedures and EXECUTE BLOCK without RETURNS should not be allowed to use SUSPEND. --- examples/empbuild/empddl.sql | 28 +++++++------------------- examples/empbuild/intlddl.sql | 38 +++++++++++------------------------ lang_helpers/gds_codes.ftn | 2 ++ lang_helpers/gds_codes.pas | 2 ++ src/dsql/StmtNodes.cpp | 6 ++++++ src/include/gen/codetext.h | 1 + src/include/gen/iberror.h | 6 ++++-- src/include/gen/msgs.h | 1 + src/include/gen/sql_code.h | 1 + src/include/gen/sql_state.h | 1 + src/msgs/facilities2.sql | 2 +- src/msgs/messages2.sql | 1 + src/msgs/system_errors2.sql | 1 + 13 files changed, 40 insertions(+), 50 deletions(-) diff --git a/examples/empbuild/empddl.sql b/examples/empbuild/empddl.sql index 2438a9d54b..d78f14646e 100644 --- a/examples/empbuild/empddl.sql +++ b/examples/empbuild/empddl.sql @@ -435,7 +435,7 @@ CREATE TABLE sales CHECK (order_status in ('new', 'open', 'shipped', 'waiting')), order_date TIMESTAMP - DEFAULT 'NOW' + DEFAULT 'NOW' NOT NULL, ship_date TIMESTAMP CHECK (ship_date >= order_date OR ship_date IS NULL), @@ -494,7 +494,7 @@ COMMIT; /**************************************************************************** * * Create stored procedures. - * + * *****************************************************************************/ @@ -541,7 +541,6 @@ BEGIN WHEN SQLCODE -530 DO EXCEPTION unknown_emp_id; END - SUSPEND; END !! @@ -605,7 +604,6 @@ BEGIN IF (any_sales > 0) THEN BEGIN EXCEPTION reassign_sales; - SUSPEND; END /* @@ -639,8 +637,6 @@ BEGIN */ DELETE FROM employee WHERE emp_no = :emp_num; - - SUSPEND; END !! @@ -841,14 +837,12 @@ BEGIN IF (ord_stat = 'shipped') THEN BEGIN EXCEPTION order_already_shipped; - SUSPEND; END /* Customer is on hold. */ ELSE IF (hold_stat = '*') THEN BEGIN EXCEPTION customer_on_hold; - SUSPEND; END /* @@ -865,12 +859,6 @@ BEGIN DO BEGIN EXCEPTION customer_check; - - UPDATE customer - SET on_hold = '*' - WHERE cust_no = :cust_no; - - SUSPEND; END /* @@ -879,8 +867,6 @@ BEGIN UPDATE sales SET order_status = 'shipped', ship_date = 'NOW' WHERE po_number = :po_num; - - SUSPEND; END !! @@ -897,7 +883,7 @@ BEGIN AND (language_req IS NOT NULL)) INTO :languages; IF (languages = ' ') THEN /* Prints 'NULL' instead of blanks */ - languages = 'NULL'; + languages = 'NULL'; i = i +1; SUSPEND; END @@ -905,16 +891,16 @@ END!! -CREATE PROCEDURE all_langs RETURNS - (code VARCHAR(5), grade VARCHAR(5), +CREATE PROCEDURE all_langs RETURNS + (code VARCHAR(5), grade VARCHAR(5), country VARCHAR(15), LANG VARCHAR(15)) AS BEGIN - FOR SELECT job_code, job_grade, job_country FROM job + FOR SELECT job_code, job_grade, job_country FROM job INTO :code, :grade, :country DO BEGIN - FOR SELECT languages FROM show_langs + FOR SELECT languages FROM show_langs (:code, :grade, :country) INTO :lang DO SUSPEND; /* Put nice separators between rows */ diff --git a/examples/empbuild/intlddl.sql b/examples/empbuild/intlddl.sql index 19f77d2a89..342a23f8ba 100644 --- a/examples/empbuild/intlddl.sql +++ b/examples/empbuild/intlddl.sql @@ -47,7 +47,7 @@ CREATE DOMAIN lastname AS VARCHAR(25) COLLATE FR_FR; CREATE DOMAIN phonenumber AS VARCHAR(20); -CREATE DOMAIN countryname AS VARCHAR(15); +CREATE DOMAIN countryname AS VARCHAR(15); CREATE DOMAIN addressline AS VARCHAR(40); @@ -441,7 +441,7 @@ CREATE TABLE sales NOT NULL CHECK (order_status in ('new', 'open', 'shipped', 'waiting')), - order_date TIMESTAMP + order_date TIMESTAMP DEFAULT 'NOW' NOT NULL, ship_date TIMESTAMP @@ -501,7 +501,7 @@ COMMIT; /**************************************************************************** * * Create stored procedures. - * + * *****************************************************************************/ @@ -548,7 +548,6 @@ BEGIN WHEN SQLCODE -530 DO EXCEPTION unknown_emp_id; END - SUSPEND; END !! @@ -612,7 +611,6 @@ BEGIN IF (any_sales > 0) THEN BEGIN EXCEPTION reassign_sales; - SUSPEND; END /* @@ -646,8 +644,6 @@ BEGIN */ DELETE FROM employee WHERE emp_no = :emp_num; - - SUSPEND; END !! @@ -848,14 +844,12 @@ BEGIN IF (ord_stat = 'shipped') THEN BEGIN EXCEPTION order_already_shipped; - SUSPEND; END /* Customer is on hold. */ ELSE IF (hold_stat = '*') THEN BEGIN EXCEPTION customer_on_hold; - SUSPEND; END /* @@ -872,22 +866,14 @@ BEGIN DO BEGIN EXCEPTION customer_check; - - UPDATE customer - SET on_hold = '*' - WHERE cust_no = :cust_no; - - SUSPEND; END /* * Ship the order. */ UPDATE sales - SET order_status = 'shipped', ship_date = 'NOW' + SET order_status = 'shipped', ship_date = 'NOW' WHERE po_number = :po_num; - - SUSPEND; END !! @@ -904,7 +890,7 @@ BEGIN AND (language_req IS NOT NULL)) INTO :languages; IF (languages = ' ') THEN /* Prints 'NULL' instead of blanks */ - languages = 'NULL'; + languages = 'NULL'; i = i +1; SUSPEND; END @@ -912,16 +898,16 @@ END!! -CREATE PROCEDURE all_langs RETURNS - (code VARCHAR(5), grade VARCHAR(5), +CREATE PROCEDURE all_langs RETURNS + (code VARCHAR(5), grade VARCHAR(5), country VARCHAR(15), LANG VARCHAR(15)) AS BEGIN - FOR SELECT job_code, job_grade, job_country FROM job + FOR SELECT job_code, job_grade, job_country FROM job INTO :code, :grade, :country DO BEGIN - FOR SELECT languages FROM show_langs + FOR SELECT languages FROM show_langs (:code, :grade, :country) INTO :lang DO SUSPEND; /* Put nice separators between rows */ @@ -944,7 +930,7 @@ CREATE PROCEDURE FRENCH_CUST_SORT RETURNS (customer VARCHAR(40), city VARCHAR(25), country VARCHAR(15)) AS BEGIN - FOR SELECT customer, city, country + FOR SELECT customer, city, country FROM customer ORDER BY customer INTO :customer, :city, :country DO @@ -956,7 +942,7 @@ CREATE PROCEDURE GERMAN_CUST_SORT RETURNS (customer VARCHAR(40), city VARCHAR(25), country VARCHAR(15)) AS BEGIN - FOR SELECT customer, city, country + FOR SELECT customer, city, country FROM customer ORDER BY customer COLLATE DE_DE INTO :customer, :city, :country DO @@ -968,7 +954,7 @@ CREATE PROCEDURE NORWAY_CUST_SORT RETURNS (customer VARCHAR(40), city VARCHAR(25), country VARCHAR(15)) AS BEGIN - FOR SELECT customer, city, country + FOR SELECT customer, city, country FROM customer ORDER BY customer COLLATE NO_NO INTO :customer, :city, :country DO diff --git a/lang_helpers/gds_codes.ftn b/lang_helpers/gds_codes.ftn index 4cc67858fd..2fbaf00f81 100644 --- a/lang_helpers/gds_codes.ftn +++ b/lang_helpers/gds_codes.ftn @@ -1940,6 +1940,8 @@ C -- PARAMETER (GDS__cannot_read_new_blob = 335545263) INTEGER*4 GDS__dyn_no_create_priv PARAMETER (GDS__dyn_no_create_priv = 335545264) + INTEGER*4 GDS__suspend_without_returns + PARAMETER (GDS__suspend_without_returns = 335545265) INTEGER*4 GDS__gfix_db_name PARAMETER (GDS__gfix_db_name = 335740929) INTEGER*4 GDS__gfix_invalid_sw diff --git a/lang_helpers/gds_codes.pas b/lang_helpers/gds_codes.pas index e7521d3cb8..5038a4a406 100644 --- a/lang_helpers/gds_codes.pas +++ b/lang_helpers/gds_codes.pas @@ -1935,6 +1935,8 @@ const gds_cannot_read_new_blob = 335545263; isc_dyn_no_create_priv = 335545264; gds_dyn_no_create_priv = 335545264; + isc_suspend_without_returns = 335545265; + gds_suspend_without_returns = 335545265; isc_gfix_db_name = 335740929; gds_gfix_db_name = 335740929; isc_gfix_invalid_sw = 335740930; diff --git a/src/dsql/StmtNodes.cpp b/src/dsql/StmtNodes.cpp index d9c8a98763..998063b40a 100644 --- a/src/dsql/StmtNodes.cpp +++ b/src/dsql/StmtNodes.cpp @@ -7956,6 +7956,12 @@ SuspendNode* SuspendNode::dsqlPass(DsqlCompilerScratch* dsqlScratch) Arg::Gds(isc_random) << Arg::Str("SUSPEND")); } + if (dsqlScratch->outputVariables.isEmpty()) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_suspend_without_returns)); + } + if (dsqlScratch->flags & DsqlCompilerScratch::FLAG_IN_AUTO_TRANS_BLOCK) // autonomous transaction { ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-901) << diff --git a/src/include/gen/codetext.h b/src/include/gen/codetext.h index 0e65cfb0f4..472e0e17d3 100644 --- a/src/include/gen/codetext.h +++ b/src/include/gen/codetext.h @@ -966,6 +966,7 @@ static const struct { {"cannot_update_old_blob", 335545262}, {"cannot_read_new_blob", 335545263}, {"dyn_no_create_priv", 335545264}, + {"suspend_without_returns", 335545265}, {"gfix_db_name", 335740929}, {"gfix_invalid_sw", 335740930}, {"gfix_incmp_sw", 335740932}, diff --git a/src/include/gen/iberror.h b/src/include/gen/iberror.h index 4e6f933c79..da2a4f46a6 100644 --- a/src/include/gen/iberror.h +++ b/src/include/gen/iberror.h @@ -1000,6 +1000,7 @@ const ISC_STATUS isc_bind_convert = 335545261L; const ISC_STATUS isc_cannot_update_old_blob = 335545262L; const ISC_STATUS isc_cannot_read_new_blob = 335545263L; const ISC_STATUS isc_dyn_no_create_priv = 335545264L; +const ISC_STATUS isc_suspend_without_returns = 335545265L; const ISC_STATUS isc_gfix_db_name = 335740929L; const ISC_STATUS isc_gfix_invalid_sw = 335740930L; const ISC_STATUS isc_gfix_incmp_sw = 335740932L; @@ -1490,7 +1491,7 @@ const ISC_STATUS isc_trace_switch_user_only = 337182757L; const ISC_STATUS isc_trace_switch_param_miss = 337182758L; const ISC_STATUS isc_trace_param_act_notcompat = 337182759L; const ISC_STATUS isc_trace_mandatory_switch_miss = 337182760L; -const ISC_STATUS isc_err_max = 1434; +const ISC_STATUS isc_err_max = 1435; #else /* c definitions */ @@ -2460,6 +2461,7 @@ const ISC_STATUS isc_err_max = 1434; #define isc_cannot_update_old_blob 335545262L #define isc_cannot_read_new_blob 335545263L #define isc_dyn_no_create_priv 335545264L +#define isc_suspend_without_returns 335545265L #define isc_gfix_db_name 335740929L #define isc_gfix_invalid_sw 335740930L #define isc_gfix_incmp_sw 335740932L @@ -2950,7 +2952,7 @@ const ISC_STATUS isc_err_max = 1434; #define isc_trace_switch_param_miss 337182758L #define isc_trace_param_act_notcompat 337182759L #define isc_trace_mandatory_switch_miss 337182760L -#define isc_err_max 1434 +#define isc_err_max 1435 #endif diff --git a/src/include/gen/msgs.h b/src/include/gen/msgs.h index 26e9282725..3f01c5c433 100644 --- a/src/include/gen/msgs.h +++ b/src/include/gen/msgs.h @@ -969,6 +969,7 @@ Data source : @4"}, /* eds_statement */ {335545262, "cannot update old BLOB"}, /* cannot_update_old_blob */ {335545263, "cannot read from new BLOB"}, /* cannot_read_new_blob */ {335545264, "No permission for CREATE @1 operation"}, /* dyn_no_create_priv */ + {335545265, "SUSPEND could not be used without RETURNS clause in PROCEDURE or EXECUTE BLOCK"}, /* suspend_without_returns */ {335740929, "data base file name (@1) already given"}, /* gfix_db_name */ {335740930, "invalid switch @1"}, /* gfix_invalid_sw */ {335740932, "incompatible switch combination"}, /* gfix_incmp_sw */ diff --git a/src/include/gen/sql_code.h b/src/include/gen/sql_code.h index 511f5eb4f8..0a4ec20495 100644 --- a/src/include/gen/sql_code.h +++ b/src/include/gen/sql_code.h @@ -965,6 +965,7 @@ static const struct { {335545262, -402}, /* 942 cannot_update_old_blob */ {335545263, -402}, /* 943 cannot_read_new_blob */ {335545264, -901}, /* 944 dyn_no_create_priv */ + {335545265, -901}, /* 945 suspend_without_returns */ {335740929, -901}, /* 1 gfix_db_name */ {335740930, -901}, /* 2 gfix_invalid_sw */ {335740932, -901}, /* 4 gfix_incmp_sw */ diff --git a/src/include/gen/sql_state.h b/src/include/gen/sql_state.h index f7b87a865a..0861e64238 100644 --- a/src/include/gen/sql_state.h +++ b/src/include/gen/sql_state.h @@ -965,6 +965,7 @@ static const struct { {335545262, "42000"}, // 942 cannot_update_old_blob {335545263, "42000"}, // 943 cannot_read_new_blob {335545264, "42000"}, // 944 dyn_no_create_priv + {335545265, "42000"}, // 945 suspend_without_returns {335740929, "00000"}, // 1 gfix_db_name {335740930, "00000"}, // 2 gfix_invalid_sw {335740932, "00000"}, // 4 gfix_incmp_sw diff --git a/src/msgs/facilities2.sql b/src/msgs/facilities2.sql index d5fee06757..0e5509ea02 100644 --- a/src/msgs/facilities2.sql +++ b/src/msgs/facilities2.sql @@ -1,7 +1,7 @@ /* MAX_NUMBER is the next number to be used, always one more than the highest message number. */ set bulk_insert INSERT INTO FACILITIES (LAST_CHANGE, FACILITY, FAC_CODE, MAX_NUMBER) VALUES (?, ?, ?, ?); -- -('2019-12-19 12:10:00', 'JRD', 0, 945) +('2020-02-04 12:10:00', 'JRD', 0, 946) ('2015-03-17 18:33:00', 'QLI', 1, 533) ('2018-03-17 12:00:00', 'GFIX', 3, 136) ('1996-11-07 13:39:40', 'GPRE', 4, 1) diff --git a/src/msgs/messages2.sql b/src/msgs/messages2.sql index 8158f38344..465fb8c713 100644 --- a/src/msgs/messages2.sql +++ b/src/msgs/messages2.sql @@ -1052,6 +1052,7 @@ Data source : @4', NULL, NULL) ('cannot_update_old_blob', 'BLB_put_segment', 'blb.cpp', NULL, 0, 942, NULL, 'cannot update old BLOB', NULL, NULL); ('cannot_read_new_blob', 'BLB_get_segment', 'blb.cpp', NULL, 0, 943, NULL, 'cannot read from new BLOB', NULL, NULL); ('dyn_no_create_priv', NULL, 'scl.epp', NULL, 0, 944, NULL, 'No permission for CREATE @1 operation', NULL, NULL); +('suspend_without_returns', NULL, 'StmtNodes.cpp', NULL, 0, 945, NULL, 'SUSPEND could not be used without RETURNS clause in PROCEDURE or EXECUTE BLOCK', NULL, NULL); -- QLI (NULL, NULL, NULL, NULL, 1, 0, NULL, 'expected type', NULL, NULL); (NULL, NULL, NULL, NULL, 1, 1, NULL, 'bad block type', NULL, NULL); diff --git a/src/msgs/system_errors2.sql b/src/msgs/system_errors2.sql index f904a2ec09..40e506a4b3 100644 --- a/src/msgs/system_errors2.sql +++ b/src/msgs/system_errors2.sql @@ -951,6 +951,7 @@ set bulk_insert INSERT INTO SYSTEM_ERRORS (SQL_CODE, SQL_CLASS, SQL_SUBCLASS, FA (-402, '42', '000', 0, 942, 'cannot_update_old_blob', NULL, NULL) (-402, '42', '000', 0, 943, 'cannot_read_new_blob', NULL, NULL) (-901, '42', '000', 0, 944, 'dyn_no_create_priv', NULL, NULL) +(-901, '42', '000', 0, 945, 'suspend_without_returns', NULL, NULL) -- GFIX (-901, '00', '000', 3, 1, 'gfix_db_name', NULL, NULL) (-901, '00', '000', 3, 2, 'gfix_invalid_sw', NULL, NULL) From 8e6825a05c45ef9a7ed7fb85b1342cd83bd32c78 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 5 Feb 2020 00:04:46 +0000 Subject: [PATCH 265/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 2b5d3574a4..0cebd2998b 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1761 + FORMAL BUILD NUMBER:1763 */ -#define PRODUCT_VER_STRING "4.0.0.1761" -#define FILE_VER_STRING "WI-T4.0.0.1761" -#define LICENSE_VER_STRING "WI-T4.0.0.1761" -#define FILE_VER_NUMBER 4, 0, 0, 1761 +#define PRODUCT_VER_STRING "4.0.0.1763" +#define FILE_VER_STRING "WI-T4.0.0.1763" +#define LICENSE_VER_STRING "WI-T4.0.0.1763" +#define FILE_VER_NUMBER 4, 0, 0, 1763 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1761" +#define FB_BUILD_NO "1763" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 3716ec5707..10227cc8b4 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1761 +BuildNum=1763 NowAt=`pwd` cd `dirname $0` From c577971da05087d5641f824db50c2cf8e28b0677 Mon Sep 17 00:00:00 2001 From: Dmitry Yemanov Date: Fri, 7 Feb 2020 10:43:16 +0300 Subject: [PATCH 266/274] This should fix empty routine names reported during DROP operations --- src/jrd/dfw.epp | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/jrd/dfw.epp b/src/jrd/dfw.epp index ce4c7dced0..fc642c4640 100644 --- a/src/jrd/dfw.epp +++ b/src/jrd/dfw.epp @@ -587,13 +587,13 @@ static void raiseRelationInUseError(const jrd_rel* relation) raiseObjectInUseError(obj_type, obj_name); } -static void raiseRoutineInUseError(const Routine* routine) +static void raiseRoutineInUseError(const Routine* routine, const QualifiedName& name) { const string obj_type = (routine->getObjectType() == obj_udf) ? "FUNCTION" : "PROCEDURE"; const string obj_name = routine->getName().toString(); - raiseObjectInUseError(obj_type, obj_name); + raiseObjectInUseError(obj_type, obj_name.hasData() ? obj_name : name.toString()); } static void raiseTooManyVersionsError(const int obj_type, const string& obj_name) @@ -712,8 +712,6 @@ namespace jrd_tra* transaction) { SET_TDBB(tdbb); - - Jrd::Attachment* attachment = tdbb->getAttachment(); const QualifiedName name(work->dfw_name, work->dfw_package); Routine* routine; @@ -745,7 +743,7 @@ namespace if (!LCK_convert(tdbb, routine->existenceLock, LCK_EX, transaction->getLockWait())) { - raiseRoutineInUseError(routine); + raiseRoutineInUseError(routine, name); } } @@ -764,7 +762,7 @@ namespace // Do not allow to modify routine used by user requests if (routine->isUsed() && MET_routine_in_use(tdbb, routine)) { - ///raiseRoutineInUseError(routine); + ///raiseRoutineInUseError(routine, name); gds__log("Modifying %s %s which is currently in use by active user requests", Self::getTypeStr(), name.toString().c_str()); @@ -792,7 +790,7 @@ namespace if (routine->getStatement()) { if (routine->getStatement()->isActive()) - raiseRoutineInUseError(routine); + raiseRoutineInUseError(routine, name); // release the request @@ -833,6 +831,7 @@ namespace { SSHORT validBlr = FALSE; + Jrd::Attachment* attachment = tdbb->getAttachment(); MemoryPool* newPool = attachment->createPool(); try { @@ -866,7 +865,8 @@ namespace jrd_tra* transaction) { SET_TDBB(tdbb); - T* routine = NULL; + const QualifiedName name(work->dfw_name, work->dfw_package); + T* routine; switch (phase) { @@ -895,7 +895,7 @@ namespace if (!LCK_convert(tdbb, routine->existenceLock, LCK_EX, transaction->getLockWait())) { - raiseRoutineInUseError(routine); + raiseRoutineInUseError(routine, name); } } @@ -914,12 +914,10 @@ namespace if (!routine) return false; - const QualifiedName name(work->dfw_name, work->dfw_package); - // Do not allow to drop routine used by user requests if (routine->isUsed() && MET_routine_in_use(tdbb, routine)) { - ///raiseRoutineInUseError(routine); + ///raiseRoutineInUseError(routine, name); gds__log("Deleting %s %s which is currently in use by active user requests", Self::getTypeStr(), name.toString().c_str()); @@ -941,7 +939,7 @@ namespace if (routine->getStatement()->isActive()) { routine->flags = old_flags; - raiseRoutineInUseError(routine); + raiseRoutineInUseError(routine, name); } routine->releaseStatement(tdbb); From 696673bca1fe8b480abd132615ba6be9439e8e71 Mon Sep 17 00:00:00 2001 From: Ilya Eremin Date: Fri, 7 Feb 2020 17:29:09 +0300 Subject: [PATCH 267/274] Fixed CORE-6110: 64-bit transaction IDs are not stored properly in status vector Arg::Str is used instead of Arg::Num for isc_concurrent_transaction, isc_rec_in_limbo, isc_tra_state error messages because transaction number is 64-bit unsigned integer. --- src/dsql/DdlNodes.epp | 7 ++++- src/jrd/tra.cpp | 6 +++- src/jrd/vio.cpp | 69 ++++++++++++++++++++++++++++++++++--------- 3 files changed, 66 insertions(+), 16 deletions(-) diff --git a/src/dsql/DdlNodes.epp b/src/dsql/DdlNodes.epp index c254a2a749..e069160b78 100644 --- a/src/dsql/DdlNodes.epp +++ b/src/dsql/DdlNodes.epp @@ -12427,9 +12427,14 @@ void AlterDatabaseNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratc // msg 297: Concurrent ALTER DATABASE is not supported Arg::PrivateDyn status(297); + string trans_num_str; if (conflict_trans) - status << Arg::Gds(isc_concurrent_transaction) << Arg::Num(conflict_trans); + { + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + trans_num_str.printf("%" UQUADFORMAT, conflict_trans); + status << Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str); + } status_exception::raise(status); } diff --git a/src/jrd/tra.cpp b/src/jrd/tra.cpp index 34e8ac8415..74697aafcc 100644 --- a/src/jrd/tra.cpp +++ b/src/jrd/tra.cpp @@ -1163,8 +1163,12 @@ jrd_tra* TRA_reconnect(thread_db* tdbb, const UCHAR* id, USHORT length) USHORT flags = 0; gds__msg_lookup(NULL, JRD_BUGCHK, message, sizeof(text), text, &flags); + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + string trans_num_str; + trans_num_str.printf("%" UQUADFORMAT, number); + ERR_post(Arg::Gds(isc_no_recon) << - Arg::Gds(isc_tra_state) << Arg::Num(number) << Arg::Str(text)); + Arg::Gds(isc_tra_state) << Arg::Str(trans_num_str) << Arg::Str(text)); } MemoryPool* const pool = attachment->createPool(); diff --git a/src/jrd/vio.cpp b/src/jrd/vio.cpp index 0a7ff9d2d4..26f474945f 100644 --- a/src/jrd/vio.cpp +++ b/src/jrd/vio.cpp @@ -873,9 +873,13 @@ bool VIO_chase_record_version(thread_db* tdbb, record_param* rpb, { tdbb->bumpRelStats(RuntimeStatistics::RECORD_CONFLICTS, relation->rel_id); + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + string trans_num_str; + trans_num_str.printf("%" UQUADFORMAT, rpb->rpb_transaction_nr); + ERR_post(Arg::Gds(isc_deadlock) << Arg::Gds(isc_read_conflict) << - Arg::Gds(isc_concurrent_transaction) << Arg::Num(rpb->rpb_transaction_nr)); + Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str)); } // refetch the record and try again. The active transaction @@ -998,7 +1002,11 @@ bool VIO_chase_record_version(thread_db* tdbb, record_param* rpb, if (!(transaction->tra_flags & TRA_ignore_limbo)) { CCH_RELEASE(tdbb, &rpb->getWindow(tdbb)); - ERR_post(Arg::Gds(isc_rec_in_limbo) << Arg::Num(rpb->rpb_transaction_nr)); + + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + string trans_num_str; + trans_num_str.printf("%" UQUADFORMAT, rpb->rpb_transaction_nr); + ERR_post(Arg::Gds(isc_rec_in_limbo) << Arg::Str(trans_num_str)); } case tra_active: @@ -1853,9 +1861,13 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) PageStack stack; if (prepare_update(tdbb, transaction, tid_fetch, rpb, &temp, 0, stack, false)) { + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + string trans_num_str; + trans_num_str.printf("%" UQUADFORMAT, rpb->rpb_transaction_nr); + ERR_post(Arg::Gds(isc_deadlock) << Arg::Gds(isc_update_conflict) << - Arg::Gds(isc_concurrent_transaction) << Arg::Num(rpb->rpb_transaction_nr)); + Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str)); } // Old record was restored and re-fetched for write. Now replace it. @@ -2603,7 +2615,12 @@ bool VIO_get_current(thread_db* tdbb, case tra_limbo: if (!(transaction->tra_flags & TRA_ignore_limbo)) - ERR_post(Arg::Gds(isc_rec_in_limbo) << Arg::Num(rpb->rpb_transaction_nr)); + { + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + string trans_num_str; + trans_num_str.printf("%" UQUADFORMAT, rpb->rpb_transaction_nr); + ERR_post(Arg::Gds(isc_rec_in_limbo) << Arg::Str(trans_num_str)); + } // fall thru case tra_active: @@ -3142,9 +3159,13 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j if (prepare_update(tdbb, transaction, org_rpb->rpb_transaction_nr, org_rpb, &temp, new_rpb, stack, false)) { + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + string trans_num_str; + trans_num_str.printf("%" UQUADFORMAT, org_rpb->rpb_transaction_nr); + ERR_post(Arg::Gds(isc_deadlock) << Arg::Gds(isc_update_conflict) << - Arg::Gds(isc_concurrent_transaction) << Arg::Num(org_rpb->rpb_transaction_nr)); + Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str)); } IDX_modify_flag_uk_modified(tdbb, org_rpb, new_rpb, transaction); @@ -3379,9 +3400,13 @@ bool VIO_refetch_record(thread_db* tdbb, record_param* rpb, jrd_tra* transaction { tdbb->bumpRelStats(RuntimeStatistics::RECORD_CONFLICTS, rpb->rpb_relation->rel_id); + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + string trans_num_str; + trans_num_str.printf("%" UQUADFORMAT, rpb->rpb_transaction_nr); + ERR_post(Arg::Gds(isc_deadlock) << Arg::Gds(isc_update_conflict) << - Arg::Gds(isc_concurrent_transaction) << Arg::Num(rpb->rpb_transaction_nr)); + Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str)); } return true; @@ -3996,12 +4021,19 @@ bool VIO_writelock(thread_db* tdbb, record_param* org_rpb, jrd_tra* transaction) org_rpb->rpb_runtime_flags |= RPB_refetch; return false; case PREPARE_LOCKERR: - // We got some kind of locking error (deadlock, timeout or lock_conflict) - // Error details should be stuffed into status vector at this point - // hvlad: we have no details as TRA_wait has already cleared the status vector - ERR_post(Arg::Gds(isc_deadlock) << - Arg::Gds(isc_update_conflict) << - Arg::Gds(isc_concurrent_transaction) << Arg::Num(org_rpb->rpb_transaction_nr)); + { + // We got some kind of locking error (deadlock, timeout or lock_conflict) + // Error details should be stuffed into status vector at this point + // hvlad: we have no details as TRA_wait has already cleared the status vector + + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + string trans_num_str; + trans_num_str.printf("%" UQUADFORMAT, org_rpb->rpb_transaction_nr); + + ERR_post(Arg::Gds(isc_deadlock) << + Arg::Gds(isc_update_conflict) << + Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str)); + } } // Old record was restored and re-fetched for write. Now replace it. @@ -5695,13 +5727,22 @@ static int prepare_update( thread_db* tdbb, { tdbb->bumpRelStats(RuntimeStatistics::RECORD_CONFLICTS, relation->rel_id); + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + string trans_num_str; + trans_num_str.printf("%" UQUADFORMAT, update_conflict_trans); + ERR_post(Arg::Gds(isc_update_conflict) << - Arg::Gds(isc_concurrent_transaction) << Arg::Num(update_conflict_trans)); + Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str)); } case tra_limbo: if (!(transaction->tra_flags & TRA_ignore_limbo)) - ERR_post(Arg::Gds(isc_rec_in_limbo) << Arg::Num(rpb->rpb_transaction_nr)); + { + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + string trans_num_str; + trans_num_str.printf("%" UQUADFORMAT, rpb->rpb_transaction_nr); + ERR_post(Arg::Gds(isc_rec_in_limbo) << Arg::Str(trans_num_str)); + } // fall thru case tra_active: From 57e703541153d6a0beff7b5d27bbf05c4daae8ed Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 7 Feb 2020 20:36:20 +0300 Subject: [PATCH 268/274] Added a sample of authentication plugin based on shared secret key --- builds/posix/Makefile.in.examples | 5 + configure.ac | 1 + examples/extauth/ExtAuth.cpp | 452 ++++++++++++++++++++++++++++++ examples/extauth/INSTALL | 64 +++++ examples/extauth/Makefile | 78 ++++++ examples/extauth/TcWrapper.cpp | 122 ++++++++ examples/extauth/TcWrapper.h | 210 ++++++++++++++ examples/extauth/keygen.cpp | 73 +++++ examples/readme | 45 ++- 9 files changed, 1048 insertions(+), 2 deletions(-) create mode 100644 examples/extauth/ExtAuth.cpp create mode 100644 examples/extauth/INSTALL create mode 100644 examples/extauth/Makefile create mode 100644 examples/extauth/TcWrapper.cpp create mode 100644 examples/extauth/TcWrapper.h create mode 100644 examples/extauth/keygen.cpp diff --git a/builds/posix/Makefile.in.examples b/builds/posix/Makefile.in.examples index 71cb426a5c..590e583914 100644 --- a/builds/posix/Makefile.in.examples +++ b/builds/posix/Makefile.in.examples @@ -98,6 +98,11 @@ $(FIREBIRD)/examples/README: $(CP) $(ROOT)/examples/*.* $(FIREBIRD)/examples/ $(CP) $(ROOT)/examples/api/*.* $(FIREBIRD)/examples/api/ $(CP) $(ROOT)/examples/dbcrypt/*.* $(FIREBIRD)/examples/dbcrypt/ + $(CP) $(ROOT)/examples/extauth/* $(FIREBIRD)/examples/extauth/ +ifeq ($(TOMCRYPT_BUILD_FLG),Y) + mkdir -p $(FIREBIRD)/examples/extauth/tomcrypt.include + $(CP) $(ROOT)/extern/libtomcrypt/src/headers/* $(FIREBIRD)/examples/extauth/tomcrypt.include +endif $(CP) $(ROOT)/examples/include/*.* $(FIREBIRD)/examples/include/ $(CP) $(ROOT)/examples/interfaces/*.* $(FIREBIRD)/examples/interfaces/ $(CP) $(ROOT)/examples/package/*.* $(FIREBIRD)/examples/package/ diff --git a/configure.ac b/configure.ac index c0e1398f62..1193e3550e 100644 --- a/configure.ac +++ b/configure.ac @@ -1216,6 +1216,7 @@ dnl # output mkdir -p gen/\$fb_tgt/firebird/examples/api mkdir -p gen/\$fb_tgt/firebird/examples/dbcrypt mkdir -p gen/\$fb_tgt/firebird/examples/empbuild + mkdir -p gen/\$fb_tgt/firebird/examples/extauth mkdir -p gen/\$fb_tgt/firebird/examples/include mkdir -p gen/\$fb_tgt/firebird/examples/interfaces mkdir -p gen/\$fb_tgt/firebird/examples/package diff --git a/examples/extauth/ExtAuth.cpp b/examples/extauth/ExtAuth.cpp new file mode 100644 index 0000000000..3e1ea6335d --- /dev/null +++ b/examples/extauth/ExtAuth.cpp @@ -0,0 +1,452 @@ +/* + * Simple shared-key based authentication plugin + * Each node (firebird server) contains same key which is used to authenticate cross-server + * connections. Each connection coming from one node to another has on target same + * login as it was on source node. + * + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * https://www.firebirdsql.org/en/initial-developer-s-public-license-version-1-0/ + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Alexander Peshkoff + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2020 Alexander Peshkoff + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + */ + +#include +#include + +#include "TcWrapper.h" + +#define HANDSHAKE_DEBUG(A) + +const unsigned LOGINSIZE = 128u; +const unsigned RANDSIZE = 32u; +const unsigned SALTLEN = 8u; + +typedef unsigned int ULong; + +using namespace std; + +namespace { + +IMaster* master = NULL; + +class PluginModule : public IPluginModuleImpl +{ +public: + PluginModule() + : pluginManager(NULL) + { } + + ~PluginModule() + { + if (pluginManager) + { + pluginManager->unregisterModule(this); + doClean(); + } + } + + void registerMe(IPluginManager* m) + { + pluginManager = m; + pluginManager->registerModule(this); + } + + void doClean() + { + pluginManager = NULL; + } + + void threadDetach() + { } + +private: + IPluginManager* pluginManager; +}; + + +template +class Factory : public IPluginFactoryImpl, ThrowStatusWrapper> +{ +public: + // IPluginFactory implementation + IPluginBase* createPlugin(ThrowStatusWrapper* status, IPluginConfig* factoryParameter) + { + IPluginBase* p = new P(status, factoryParameter); + p->addRef(); + return p; + } +}; + +// +// Common RSA helper +// + +class PluginData +{ +public: + PluginData(ThrowStatusWrapper* status, IPluginConfig* cnf) + : refCounter(0), owner(NULL), iniLvl(0) + { + hash.init(status); + iniLvl = 1; + pseudoRand.init(status); + iniLvl = 2; + + AutoRelease conf(cnf->getDefaultConfig(status)); + if (!conf) + return; + AutoRelease ce(conf->find(status, "Key")); + if (!ce) + return; + + // import a key + unsigned char key[4096]; + unsigned keySize = readHexKey(status, ce->getValue(), key, sizeof(key)); + check(status, rsa_import(key, keySize, &privateKey), + "ExtAuth plugin failed to initialize - error importing private RSA key"); + iniLvl = 3; + } + + ~PluginData() + { + if (iniLvl >= 3) + rsa_free(&privateKey); + if (iniLvl >= 2) + pseudoRand.fini(); + if (iniLvl >= 1) + hash.fini(); + } + +protected: + atomic refCounter; + IReferenceCounted* owner; + + PseudoRandom pseudoRand; + HashSha256 hash; + rsa_key privateKey; + int iniLvl; +}; + + +// +// Client plugin +// + +class ExtAuthClient : public IClientImpl, public PluginData +{ +public: + ExtAuthClient(ThrowStatusWrapper* status, IPluginConfig* cnf) + : PluginData(status, cnf), + ignorePassword(false), + ignoreLogin(false) + { + AutoRelease conf(cnf->getDefaultConfig(status)); + if (conf) + { + AutoRelease igPass(conf->find(status, "IgnorePassword")); + if (igPass) + ignorePassword = igPass->getBoolValue(); + AutoRelease igLgn(conf->find(status, "IgnoreLogin")); + if (igLgn) + ignoreLogin = igLgn->getBoolValue(); + } + } + + // IClient implementation + int authenticate(ThrowStatusWrapper* status, IClientBlock* cBlock); + + int release() + { + if (--refCounter == 0) + { + delete this; + return 0; + } + return 1; + } + + void addRef() + { + ++refCounter; + } + + void setOwner(IReferenceCounted* o) + { + owner = o; + } + + IReferenceCounted* getOwner() + { + return owner; + } + +private: + bool ignorePassword, ignoreLogin; +}; + +int ExtAuthClient::authenticate(ThrowStatusWrapper* status, IClientBlock* cBlock) +{ + try + { + // did we initialize correctly? + if (iniLvl < 3) + return AUTH_CONTINUE; + + // check for missing login from the user + if ((!ignoreLogin) && cBlock->getLogin()) + return AUTH_CONTINUE; + + // check for missing password from the user + if ((!ignorePassword) && cBlock->getPassword()) + return AUTH_CONTINUE; + + // check for presence of authenticatiion block + IAuthBlock* authBlock = cBlock->getAuthBlock(status); + if (!authBlock) + return AUTH_CONTINUE; + if (!authBlock->first(status)) + return AUTH_CONTINUE; + + // and for presence of user name in that authenticatiion block + const char* login = NULL; + do + { + const char* type = authBlock->getType(); + if (type && (strcmp(type, "USER") == 0)) + { + login = authBlock->getName(); + if (login) + break; + } + } while(authBlock->next(status)); + if (!login) + return AUTH_CONTINUE; + + // check if server started to talk to us + unsigned dl = 0; + const unsigned char* data = cBlock->getData(&dl); + if (dl == 0 || !data) + return AUTH_MORE_DATA; + + // decrypt message + unsigned char bytes[RANDSIZE + LOGINSIZE + 1]; + unsigned long outlen = RANDSIZE; + int result = 0; + check(status, rsa_decrypt_key(data, dl, bytes, &outlen, NULL, 0, hash.index, &result, &privateKey), + "Error decrypting message"); + if (outlen < RANDSIZE) + error(status, "Malformed data from server - missing random block"); + + // next append login to random block + unsigned len = strlen(login); + if (len > LOGINSIZE) + len = LOGINSIZE; + memcpy(&bytes[RANDSIZE], login, len); + + // calc hash for whole block + hash_state state; + sha256_init(&state); + check(status, sha256_process(&state, bytes, RANDSIZE + len), "Error hashing message"); + unsigned char digest[256 / 8]; + check(status, sha256_done(&state, digest), "Error extracting hash"); + + // build message + unsigned char msg[4096]; + + // put login to it + memcpy(msg, login, len); + msg[len++] = 0; + + // append sign of hash to it + unsigned long signLen = sizeof(msg) - len; + unsigned char* sign = &msg[len]; + check(status, rsa_sign_hash(digest, sizeof digest, sign, &signLen, &pseudoRand.state, + pseudoRand.index, hash.index, SALTLEN, &privateKey), "Error signing message hash"); + + // send message + cBlock->putData(status, len + signLen, msg); + + // output the wire crypt key + ICryptKey* cKey = cBlock->newKey(status); + cKey->setSymmetric(status, "Symmetric", RANDSIZE, bytes); + HANDSHAKE_DEBUG( fprintf(stderr, "Key ="); for (unsigned n = 0; n < RANDSIZE; ++n) + fprintf(stderr, " %02u", bytes[n]); fprintf(stderr, "\n"); ) + + return AUTH_SUCCESS; + } + catch(const FbException& ex) + { + status->setErrors(ex.getStatus()->getErrors()); + return AUTH_FAILED; + } +} + + +// +// Server plugin +// + +class ExtAuthServer : public IServerImpl, public PluginData +{ +public: + ExtAuthServer(ThrowStatusWrapper* status, IPluginConfig* cnf) + : PluginData(status, cnf), sentData(false) + { } + + // IServer implementation + int authenticate(ThrowStatusWrapper* status, IServerBlock* sBlock, IWriter* writerInterface); + + void setDbCryptCallback(ThrowStatusWrapper* status, ICryptKeyCallback* cryptCallback) + { } + + int release() + { + if (--refCounter == 0) + { + delete this; + return 0; + } + return 1; + } + + void addRef() + { + ++refCounter; + } + + void setOwner(IReferenceCounted* o) + { + owner = o; + } + + IReferenceCounted* getOwner() + { + return owner; + } + +private: + unsigned char msg[RANDSIZE + LOGINSIZE]; + bool sentData; +}; + + +int ExtAuthServer::authenticate(ThrowStatusWrapper* status, IServerBlock* sBlock, IWriter* writerInterface) +{ + try + { + // did we initialize correctly? + if (iniLvl < 3) + return AUTH_CONTINUE; + + unsigned dl = 0; + const unsigned char* data = sBlock->getData(&dl); + if (!sentData) + { + // fbassert(dl == 0 && !data); + + // build message: first of all get some randomness + pseudoRand.getDsc()->read(msg, RANDSIZE, &pseudoRand.state); + + // now encrypt that random block + unsigned char encrypted[4096]; + unsigned long encLen = sizeof encrypted; + check(status, rsa_encrypt_key(msg, RANDSIZE, encrypted, &encLen, NULL, 0, + &pseudoRand.state, pseudoRand.index, hash.index, &privateKey), "Error encrypting message"); + + // send message + sBlock->putData(status, encLen, encrypted); + sentData = true; + + return AUTH_MORE_DATA; + } + + // decompose message + const char* login = reinterpret_cast(data); + unsigned len = strnlen(login, dl); + if (len == dl) + error(status, "Wrong data from client - no signature in a message"); + if (len == 0) + error(status, "Wrong data from client - empty login"); + if (len > LOGINSIZE) + error(status, "Wrong data from client - login too long"); + memcpy(&msg[RANDSIZE], data, len); + const unsigned char* sign = &data[len + 1]; + unsigned long signLen = dl - (len + 1); + + // calc hash for message + hash_state state; + sha256_init(&state); + check(status, sha256_process(&state, msg, RANDSIZE + len), "Error hashing message"); + unsigned char digest[256 / 8]; + check(status, sha256_done(&state, digest), "Error extracting hash"); + + // validate signature + int result = 0; + int err = rsa_verify_hash(sign, signLen, digest, sizeof digest, hash.index, SALTLEN, &result, &privateKey); + if (err != CRYPT_INVALID_PACKET) + check(status, err, "Error verifying digital signature"); + else + result = 0; + if (!result) + error(status, "Malformed data from client - invalid digital signature"); + + // output the wire crypt key + ICryptKey* cKey = sBlock->newKey(status); + cKey->setSymmetric(status, "Symmetric", RANDSIZE, msg); + HANDSHAKE_DEBUG( fprintf(stderr, "Key ="); for (unsigned n = 0; n < RANDSIZE; ++n) + fprintf(stderr, " %02x", msg[n]); fprintf(stderr, "\n"); ) + + // store received login name in auth block + writerInterface->add(status, login); + + return AUTH_SUCCESS; + } + catch(const FbException& ex) + { + status->setErrors(ex.getStatus()->getErrors()); + } + return AUTH_FAILED; +} + + +// +// Static variables +// + +PluginModule module; +Factory clientFactory; +Factory serverFactory; + +} // anonymous namespace + + +#if defined(_WIN32) +#define FB_DLL_EXPORT __declspec(dllexport) +#else +#define FB_DLL_EXPORT +#endif + +extern "C" void FB_DLL_EXPORT FB_PLUGIN_ENTRY_POINT(IMaster* m) +{ + master = m; + IPluginManager* pluginManager = master->getPluginManager(); + + module.registerMe(pluginManager); + pluginManager->registerPluginFactory(IPluginManager::TYPE_AUTH_CLIENT, "ExtAuth", &clientFactory); + pluginManager->registerPluginFactory(IPluginManager::TYPE_AUTH_SERVER, "ExtAuth", &serverFactory); +} diff --git a/examples/extauth/INSTALL b/examples/extauth/INSTALL new file mode 100644 index 0000000000..e02d1c1378 --- /dev/null +++ b/examples/extauth/INSTALL @@ -0,0 +1,64 @@ +0. Brief description. + +ExtAuth plugin is useful when you want to run 'execute statement on external' statement +connecting to databases on non-local servers but do not wish to explicitly add login and +password to your PL/SQL code. When 2 servers completely trust each other this plugin may +be used to enable access to remote database without entering login and password in SQL +code. To ensure that connection comes from trusted source shared secret key (placed into +plugin's .conf file) is used. That means that the value of a "Key" parameter should be +exacly the same for all trusting each other hosts. Pay attention - SQL name of connected +user on remote host may not match local logon, it depends also upon mappings on remote +host. + +1. Before starting the build. + +This authentication plugin is using TomCrypt (https://www.libtom.net/LibTomCrypt/) library. +Firebird since v.4 is actively using it. Depending upon build type tomcrypt binary may be +included or not included into your package. In a case when it's included you will find +appropriate H-files in tomcrypt.include subdir. If not that means that native OS library +is used and you should also work with it. Depending upon your OS you will may be need +to install development package for tomcrypt. + +2. Building plugin. + +Type 'make' in this directory. Build system supposes that it was not moved out of standard +firebird tree. In a case when you did it manually you will sooner of all have to change +Makefile appropriately. If you use firebird with different operating systems and/or hardware +you should build plugin for each used configuration. + +3. Installing plugin. + +Makefile has install target (make install) which may be used for current box. However +this is not full solution because plugin is supposed to be used to connect at least two +separate servers. See 'Testing' for more details. + +4.Testing. + +- imagine you have to hosts: host1 and host2; +- generate configuration file using extauth_keygen utility on any of them (only ONCE - +on ONE host !!!); +- copy that file and plugin itself to $FIREBIRD/plugins directory on each host; +- modife firebird.cond, it should contain something like: + AuthServer = Srp256, ExtAuth + AuthClient = Srp256, ExtAuth +lines, certainly something else may be used instead recommended Srp256; +- if you need WIN_SSPI plugin please add it AFTER ExtAuth; +- do not forget to restart firebird after reconfiguring it; +- create minimal required mapping on host1: + CREATE MAPPING EXT USING PLUGIN EXTAUTH FROM ANY USER TO USER EXTUSER; +- run the following script on host2: + ^SET TERM ^; + EXECUTE BLOCK RETURNS(REMNAME CHAR(32)) AS BEGIN + EXECUTE STATEMENT 'SELECT CURRENT_USER FROM RDB$DATABASE' + ON EXTERNAL 'host1:employee' INTO :REMNAME; + SUSPEND; + END^ + SET TERM ;^ +you should get something like this: + REMNAME + ============================== + EXTUSER +- explicitly specifying login and/or password in SQL statement normally deactivates +this plugin but one can use IgnoreLogin and IgnorePassword parameters to change that. + + diff --git a/examples/extauth/Makefile b/examples/extauth/Makefile new file mode 100644 index 0000000000..b5fbff9327 --- /dev/null +++ b/examples/extauth/Makefile @@ -0,0 +1,78 @@ +# The contents of this file are subject to the Initial +# Developer's Public License Version 1.0 (the "License"); +# you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# https://www.firebirdsql.org/en/initial-developer-s-public-license-version-1-0/ +# +# Software distributed under the License is distributed AS IS, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. +# See the License for the specific language governing rights +# and limitations under the License. +# +# The Original Code was created by Alexander Peshkoff +# for the Firebird Open Source RDBMS project. +# +# Copyright (c) 2020 Alexander Peshkoff +# and all contributors signed below. +# +# All Rights Reserved. +# Contributor(s): ______________________________________. + +ROOT=../.. +#ROOT=$(shell cd ../..; pwd) + +PLUGINS=$(ROOT)/plugins +BIN=$(ROOT)/bin +LIB=$(ROOT)/lib + +LIB_PREFIX=lib +SHRLIB_EXT=so + +makePluginName=$(PLUGINS)/$(LIB_PREFIX)$(1).$(SHRLIB_EXT) + +TOMCRYPT_COMPILE=-DLTC_PTHREAD -DUSE_LTM -DLTM_DESC +OwnInclude=$(shell [ -d tomcrypt.include ] && echo Yes || echo No) +ifeq ($(OwnInclude), Yes) +TOMCRYPT_COMPILE += -Itomcrypt.include +TOMCRYPT_LINK=-L$(LIB)/.tm +endif +BLD_SIMPLE_KEY_AUTH=libExtAuth.so +SIMPLE_KEY_AUTH=$(call makePluginName,ExtAuth) +BLD_KEYGEN=extauth_keygen +KEYGEN=$(BIN)/$(BLD_KEYGEN) + +KEYGEN_objects=keygen.o +TCWRAP_objects=TcWrapper.o +KEY_AUTH_objects=ExtAuth.o + +CXXFLAGS=-std=c++11 -pthread -I$(ROOT)/include -fPIC $(TOMCRYPT_COMPILE) +LDFLAGS=-pthread -L$(LIB) -Wl,-rpath,'$$ORIGIN/../lib' $(TOMCRYPT_LINK) + +LINK=c++ +LINK_LIBS=-lfbclient -ltomcrypt + +.PHONY: all keygen plugin install + +all: keygen plugin + +keygen: $(BLD_KEYGEN) + +$(BLD_KEYGEN): $(KEYGEN_objects) $(TCWRAP_objects) + $(LINK) $(LDFLAGS) $^ -o $@ $(LINK_LIBS) + +plugin: $(BLD_SIMPLE_KEY_AUTH) + +$(BLD_SIMPLE_KEY_AUTH): $(KEY_AUTH_objects) $(TCWRAP_objects) + $(LINK) -shared $(LDFLAGS) $^ -o $@ $(LINK_LIBS) + +clean: + rm -f *.o* $(BLD_KEYGEN) $(BLD_SIMPLE_KEY_AUTH) + +install: $(SIMPLE_KEY_AUTH) $(KEYGEN) + +$(SIMPLE_KEY_AUTH): $(BLD_SIMPLE_KEY_AUTH) + cp $^ $@ + +$(KEYGEN): $(BLD_KEYGEN) + cp $^ $@ + diff --git a/examples/extauth/TcWrapper.cpp b/examples/extauth/TcWrapper.cpp new file mode 100644 index 0000000000..d4feacebb9 --- /dev/null +++ b/examples/extauth/TcWrapper.cpp @@ -0,0 +1,122 @@ +/* + * Tomcrypt library <= firebird : c++ wrapper. + * + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * https://www.firebirdsql.org/en/initial-developer-s-public-license-version-1-0/ + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Alexander Peshkoff + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2020 Alexander Peshkoff + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + */ + +#include "TcWrapper.h" + +namespace { +// LTC hack +class GInit +{ +public: + GInit() + { + ltc_mp = ltm_desc; + } +}; +GInit gInit; +} + +void error(ThrowStatusWrapper* status, const char* text) +{ + if (! status) + throw text; + + ISC_STATUS_ARRAY v; + v[0] = isc_arg_gds; + v[1] = isc_random; + v[2] = isc_arg_string; + v[3] = (ISC_STATUS) text; + v[4] = isc_arg_end; + + throw FbException(status, v); +} + +void check(ThrowStatusWrapper* status, int err, const char* text) +{ + if (err == CRYPT_OK) + return; + + char buf[256]; + sprintf(buf, "%s: %s", text, error_to_string(err)); + error(status, buf); +} + +unsigned readHexKey(ThrowStatusWrapper* status, const char* hex, unsigned char* key, unsigned bufSize) +{ + unsigned char* k = key; + const char* const end = hex + strlen(hex) - 1; + for (const char* s = hex; s < end; s += 2) + { + if (k - key >= bufSize) + break; + + // FF + char ss[3]; + ss[0] = s[0]; + ss[1] = s[1]; + ss[2] = 0; + unsigned c = strtoul(ss, NULL, 16); + if (c > 255) + error(status, "Key format error"); + *k++ = static_cast(c); + } + return k - key; +} + +void PseudoRandom::init(ThrowStatusWrapper* status) +{ + // LTC hack + ltc_mp = ltm_desc; + + // register yarrow + index = register_prng(&yarrow_desc); + if (index == -1) + error(status, "Error registering PRNG yarrow"); + + // setup the PRNG + check(status, yarrow_start(&state), "Error starting PRNG yarrow"); + check(status, rng_make_prng(64, index, &state, NULL), "Error setting up PRNG yarrow"); +} + +void PseudoRandom::fini() +{ + yarrow_done(&state); +} + +const PseudoRandom::PrngDescriptor* PseudoRandom::getDsc() +{ + return &yarrow_desc; +} + +void Hash::init(ThrowStatusWrapper* status, const ltc_hash_descriptor* desc) +{ + // LTC hack + ltc_mp = ltm_desc; + + /* register SHA256 */ + index = register_hash(desc); + if (index == -1) + error(status, "Error registering SHA256"); +} + diff --git a/examples/extauth/TcWrapper.h b/examples/extauth/TcWrapper.h new file mode 100644 index 0000000000..14f9e1826e --- /dev/null +++ b/examples/extauth/TcWrapper.h @@ -0,0 +1,210 @@ +/* + * Tomcrypt library <= firebird : c++ wrapper. + * + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * https://www.firebirdsql.org/en/initial-developer-s-public-license-version-1-0/ + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Alexander Peshkoff + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2020 Alexander Peshkoff + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + */ + +#include + +#include +using namespace Firebird; + +#include + +void error(ThrowStatusWrapper* status, const char* text); +void check(ThrowStatusWrapper* status, int err, const char* text); +unsigned readHexKey(ThrowStatusWrapper* status, const char* hex, unsigned char* key, unsigned bufSize); + +class PseudoRandom +{ +public: +#if CRYPT > 0x0100 + typedef ltc_prng_descriptor PrngDescriptor; +#else + typedef _prng_descriptor PrngDescriptor; +#endif + + void init(ThrowStatusWrapper* status); + void fini(); + const PrngDescriptor* getDsc(); + + int index; + prng_state state; +}; + +class Hash +{ +protected: + void init(ThrowStatusWrapper* status, const ltc_hash_descriptor* desc); + +public: + void fini() + { } + + int index; +}; + +class HashSha1 : public Hash +{ +public: + void init(ThrowStatusWrapper* status) + { + Hash::init(status, &sha1_desc); + } +}; + +class HashSha256 : public Hash +{ +public: + void init(ThrowStatusWrapper* status) + { + Hash::init(status, &sha256_desc); + } +}; + +// controls reference counter of the object where points +enum NoIncrement {NO_INCREMENT}; +template +class RefPtr +{ +public: + RefPtr() : ptr(NULL) + { } + + explicit RefPtr(T* p) : ptr(p) + { + if (ptr) + { + ptr->addRef(); + } + } + + // This special form of ctor is used to create refcounted ptr from interface, + // returned by a function (which increments counter on return) + RefPtr(NoIncrement x, T* p) : ptr(p) + { } + + RefPtr(const RefPtr& r) : ptr(r.ptr) + { + if (ptr) + { + ptr->addRef(); + } + } + + ~RefPtr() + { + if (ptr) + { + ptr->release(); + } + } + + T* operator=(T* p) + { + return assign(p); + } + + T* operator=(const RefPtr& r) + { + return assign(r.ptr); + } + + operator T*() + { + return ptr; + } + + T* operator->() + { + return ptr; + } + + operator const T*() const + { + return ptr; + } + + const T* operator->() const + { + return ptr; + } + + bool operator !() const + { + return !ptr; + } + + bool operator ==(const RefPtr& r) const + { + return ptr == r.ptr; + } + + bool operator !=(const RefPtr& r) const + { + return ptr != r.ptr; + } + + void clear() throw() // Used after detach/commit/close/etc., i.e. release() not needed + { + ptr = NULL; + } + + void assignNoIncrement(T* const p) + { + assign(NULL); + ptr = p; + } + +private: + T* assign(T* const p) + { + if (ptr != p) + { + if (p) + { + p->addRef(); + } + + T* tmp = ptr; + ptr = p; + + if (tmp) + { + tmp->release(); + } + } + + return ptr; + } + + T* ptr; +}; + +// Often used form of RefPtr +template +class AutoRelease : public RefPtr +{ +public: + AutoRelease(R* r) + : RefPtr(NO_INCREMENT, r) + { } +}; diff --git a/examples/extauth/keygen.cpp b/examples/extauth/keygen.cpp new file mode 100644 index 0000000000..b7061d0d78 --- /dev/null +++ b/examples/extauth/keygen.cpp @@ -0,0 +1,73 @@ +/* + * RSA key generate. + * + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * https://www.firebirdsql.org/en/initial-developer-s-public-license-version-1-0/ + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Alexander Peshkoff + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2020 Alexander Peshkoff + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + */ + +#include "TcWrapper.h" + +int main(int ac, char** av) +{ + try + { + int len = ac > 1 ? atoi(av[1]) : 2048; + + PseudoRandom pseudoRand; + pseudoRand.init(NULL); + + rsa_key key; + check(NULL, rsa_make_key(&pseudoRand.state, pseudoRand.index, len / 8, 65537, &key), + "Error making RSA key"); + + unsigned char outbuf[4096]; + unsigned long outlen = sizeof outbuf; + check(NULL, rsa_export(outbuf, &outlen, PK_PRIVATE, &key), + "Error exporting private RSA key"); + + const char* const file = "ExtAuth.conf"; + FILE* conf = fopen(file, "w"); + if (!conf) + { + perror(file); + return 1; + } + + fprintf(conf, "Key = "); + for (unsigned i = 0; i < outlen; ++i) + fprintf(conf, "%02x", outbuf[i]); + fprintf(conf, "\n#IgnoreLogin = No\n#IgnorePassword = No\n"); + + if (fclose(conf) != 0) + { + perror(file); + return 1; + } + printf("Wrote configuration file %s\n", file); + } + catch (const char* message) + { + fprintf(stderr, "%s\n", message); + return 1; + } + + return 0; +} + diff --git a/examples/readme b/examples/readme index e53afb2392..b79d9bb03c 100644 --- a/examples/readme +++ b/examples/readme @@ -65,7 +65,8 @@ stat12.e Event wait and signaling. stat12t.e WHENEVER SQLERROR and BASED_ON clause are illustrated by many programs. -^L + + Embedded Dynamic SQL @@ -84,7 +85,8 @@ dyn5.e Demonstrate dynamic reallocation of SQLDA and 'describe' statement. dynfull.e A full_dsql program (process unknown statements). VARY struct is used by dyn3.e, dynfull.e. -^L + + API Interface @@ -140,3 +142,42 @@ SQLCODE extraction from status is covered by several programs. Zero transaction handle is covered in several programs, ex. api14.e. + +Object-oriented API (interfaces) + +Program Description +--------- -------------------------------------------------------- +01.create.cpp / A sample of creating new database and new table in it. +01.create.pas Has pascal (delphi) implementation. + +02.update.cpp UPDATE statement with parameters. + +03.select.cpp SELECT statement without input parameters. + +04.print_table.cpp Open cursor from attachment and access blob data. + +05.user_metadata.cpp Cursor with user-implemented interface of message format. + +06.fb_message.cpp Use of static messages and decfloat values. + +07.blob.cpp Loading data into blob and reading it. + +08.events.cpp Working with events. + +09.service.cpp Using services: prints server version and db statistics. + +10.backup.cpp Using services: backup database. + +11.batch.cpp Working with batch interface. + +12.batch_isc.cpp Working with batch interface from ISC API. + + +FbSampleAtomic typedef required in many examples. + + + +dbcrypt - a sample of XOR database encryption (do not use in production!!!) + +extauth - authentication for cross-server connections based on having same secret key on all servers + From 83d4b25f2bb38fb3ed1d920c56778e043e21c4cb Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Sat, 8 Feb 2020 00:04:39 +0000 Subject: [PATCH 269/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 0cebd2998b..22bd21bd76 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1763 + FORMAL BUILD NUMBER:1766 */ -#define PRODUCT_VER_STRING "4.0.0.1763" -#define FILE_VER_STRING "WI-T4.0.0.1763" -#define LICENSE_VER_STRING "WI-T4.0.0.1763" -#define FILE_VER_NUMBER 4, 0, 0, 1763 +#define PRODUCT_VER_STRING "4.0.0.1766" +#define FILE_VER_STRING "WI-T4.0.0.1766" +#define LICENSE_VER_STRING "WI-T4.0.0.1766" +#define FILE_VER_NUMBER 4, 0, 0, 1766 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1763" +#define FB_BUILD_NO "1766" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 10227cc8b4..1233f292f5 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1763 +BuildNum=1766 NowAt=`pwd` cd `dirname $0` From d99fcf9044d15b89ac4e675687695059fe0a09a7 Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Tue, 11 Feb 2020 16:10:38 +0000 Subject: [PATCH 270/274] Fix "bug" CORE-6243 - Regression: v4 Beta 1 rejects POSITION element of v2.5 defined SQL2003 CREATE TRIGGER syntax. --- src/dsql/Parser.h | 10 ++++++++++ src/dsql/parse.y | 9 +++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/dsql/Parser.h b/src/dsql/Parser.h index f74235a52e..438df9d990 100644 --- a/src/dsql/Parser.h +++ b/src/dsql/Parser.h @@ -286,6 +286,16 @@ private: clause = value; } + template + void setClause(BaseNullable& clause, const char* duplicateMsg, const BaseNullable& value) + { + if (value.specified) + { + checkDuplicateClause(clause, duplicateMsg); + clause = value.value; + } + } + template void setClause(NestConst& clause, const char* duplicateMsg, const T2& value) { diff --git a/src/dsql/parse.y b/src/dsql/parse.y index 1ee02446c2..ed4b4ef301 100644 --- a/src/dsql/parse.y +++ b/src/dsql/parse.y @@ -3775,14 +3775,14 @@ create_trigger_common($trigger) { $trigger->active = $1; $trigger->type = $2; - $trigger->position = $3; + setClause($trigger->position, "POSITION", $3); } | FOR symbol_table_name trigger_active table_trigger_type trigger_position { $trigger->relationName = *$2; $trigger->active = $3; $trigger->type = $4; - $trigger->position = $5; + setClause($trigger->position, "POSITION", $5); } ; @@ -3807,10 +3807,11 @@ trigger_active %type trigger_type() trigger_type($trigger) - : table_trigger_type ON symbol_table_name + : table_trigger_type trigger_position ON symbol_table_name { $$ = $1; - $trigger->relationName = *$3; + setClause($trigger->position, "POSITION", $2); + $trigger->relationName = *$4; } | ON trigger_db_type { $$ = $2; } From 9a7a0a3750a02558983a4ffd92fced733bf89be5 Mon Sep 17 00:00:00 2001 From: firebirds <> Date: Wed, 12 Feb 2020 00:06:05 +0000 Subject: [PATCH 271/274] increment build number --- src/jrd/build_no.h | 12 ++++++------ src/misc/writeBuildNum.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jrd/build_no.h b/src/jrd/build_no.h index 22bd21bd76..d0f102bb93 100644 --- a/src/jrd/build_no.h +++ b/src/jrd/build_no.h @@ -3,16 +3,16 @@ *** DO NOT EDIT *** TO CHANGE ANY INFORMATION IN HERE PLEASE EDIT src/misc/writeBuildNum.sh - FORMAL BUILD NUMBER:1766 + FORMAL BUILD NUMBER:1767 */ -#define PRODUCT_VER_STRING "4.0.0.1766" -#define FILE_VER_STRING "WI-T4.0.0.1766" -#define LICENSE_VER_STRING "WI-T4.0.0.1766" -#define FILE_VER_NUMBER 4, 0, 0, 1766 +#define PRODUCT_VER_STRING "4.0.0.1767" +#define FILE_VER_STRING "WI-T4.0.0.1767" +#define LICENSE_VER_STRING "WI-T4.0.0.1767" +#define FILE_VER_NUMBER 4, 0, 0, 1767 #define FB_MAJOR_VER "4" #define FB_MINOR_VER "0" #define FB_REV_NO "0" -#define FB_BUILD_NO "1766" +#define FB_BUILD_NO "1767" #define FB_BUILD_TYPE "T" #define FB_BUILD_SUFFIX "Firebird 4.0 Beta 1" diff --git a/src/misc/writeBuildNum.sh b/src/misc/writeBuildNum.sh index 1233f292f5..75f2f588ef 100755 --- a/src/misc/writeBuildNum.sh +++ b/src/misc/writeBuildNum.sh @@ -9,7 +9,7 @@ BuildType=T MajorVer=4 MinorVer=0 RevNo=0 -BuildNum=1766 +BuildNum=1767 NowAt=`pwd` cd `dirname $0` From 641eca3aa096e072883293d50908ce6249386b41 Mon Sep 17 00:00:00 2001 From: hvlad Date: Wed, 12 Feb 2020 13:08:40 +0200 Subject: [PATCH 272/274] A bit better solution for CORE-6110: 64-bit transaction IDs are not stored properly in status vector --- src/common/StatusArg.cpp | 12 +++++++++ src/common/StatusArg.h | 11 ++++++++ src/dsql/DdlNodes.epp | 4 +-- src/jrd/tra.cpp | 5 +--- src/jrd/vio.cpp | 56 +++++++++++----------------------------- 5 files changed, 40 insertions(+), 48 deletions(-) diff --git a/src/common/StatusArg.cpp b/src/common/StatusArg.cpp index 3db2b07af4..9b646b5445 100644 --- a/src/common/StatusArg.cpp +++ b/src/common/StatusArg.cpp @@ -333,6 +333,18 @@ PrivateDyn::PrivateDyn(ISC_STATUS codeWithoutFacility) throw() : Num::Num(ISC_STATUS s) throw() : Base(isc_arg_number, s) { } +Int64::Int64(SINT64 val) throw() : + Str(text) +{ + sprintf(text, "%" SQUADFORMAT, val); +} + +Int64::Int64(FB_UINT64 val) throw() : + Str(text) +{ + sprintf(text, "%" UQUADFORMAT, val); +} + Quad::Quad(const ISC_QUAD* quad) throw() : Str(text) { diff --git a/src/common/StatusArg.h b/src/common/StatusArg.h index d230c50fba..9540240c1a 100644 --- a/src/common/StatusArg.h +++ b/src/common/StatusArg.h @@ -248,6 +248,17 @@ public: explicit Num(ISC_STATUS s) throw(); }; +// On 32-bit architecture ISC_STATUS can't fit 64-bit integer therefore +// convert such a numbers into text and put string into status-vector +class Int64 : public Str +{ +public: + explicit Int64(SINT64 val) throw(); + explicit Int64(FB_UINT64 val) throw(); +private: + char text[24]; +}; + class Quad : public Str { public: diff --git a/src/dsql/DdlNodes.epp b/src/dsql/DdlNodes.epp index e069160b78..1d8ebeb8d8 100644 --- a/src/dsql/DdlNodes.epp +++ b/src/dsql/DdlNodes.epp @@ -12427,13 +12427,11 @@ void AlterDatabaseNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratc // msg 297: Concurrent ALTER DATABASE is not supported Arg::PrivateDyn status(297); - string trans_num_str; if (conflict_trans) { // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer - trans_num_str.printf("%" UQUADFORMAT, conflict_trans); - status << Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str); + status << Arg::Gds(isc_concurrent_transaction) << Arg::Int64(conflict_trans); } status_exception::raise(status); diff --git a/src/jrd/tra.cpp b/src/jrd/tra.cpp index 74697aafcc..0211b4f9af 100644 --- a/src/jrd/tra.cpp +++ b/src/jrd/tra.cpp @@ -1164,11 +1164,8 @@ jrd_tra* TRA_reconnect(thread_db* tdbb, const UCHAR* id, USHORT length) gds__msg_lookup(NULL, JRD_BUGCHK, message, sizeof(text), text, &flags); // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer - string trans_num_str; - trans_num_str.printf("%" UQUADFORMAT, number); - ERR_post(Arg::Gds(isc_no_recon) << - Arg::Gds(isc_tra_state) << Arg::Str(trans_num_str) << Arg::Str(text)); + Arg::Gds(isc_tra_state) << Arg::Int64(number) << Arg::Str(text)); } MemoryPool* const pool = attachment->createPool(); diff --git a/src/jrd/vio.cpp b/src/jrd/vio.cpp index 26f474945f..0bc1bb9bd3 100644 --- a/src/jrd/vio.cpp +++ b/src/jrd/vio.cpp @@ -874,12 +874,9 @@ bool VIO_chase_record_version(thread_db* tdbb, record_param* rpb, tdbb->bumpRelStats(RuntimeStatistics::RECORD_CONFLICTS, relation->rel_id); // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer - string trans_num_str; - trans_num_str.printf("%" UQUADFORMAT, rpb->rpb_transaction_nr); - ERR_post(Arg::Gds(isc_deadlock) << Arg::Gds(isc_read_conflict) << - Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str)); + Arg::Gds(isc_concurrent_transaction) << Arg::Int64(rpb->rpb_transaction_nr)); } // refetch the record and try again. The active transaction @@ -1004,9 +1001,7 @@ bool VIO_chase_record_version(thread_db* tdbb, record_param* rpb, CCH_RELEASE(tdbb, &rpb->getWindow(tdbb)); // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer - string trans_num_str; - trans_num_str.printf("%" UQUADFORMAT, rpb->rpb_transaction_nr); - ERR_post(Arg::Gds(isc_rec_in_limbo) << Arg::Str(trans_num_str)); + ERR_post(Arg::Gds(isc_rec_in_limbo) << Arg::Int64(rpb->rpb_transaction_nr)); } case tra_active: @@ -1862,12 +1857,9 @@ void VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) if (prepare_update(tdbb, transaction, tid_fetch, rpb, &temp, 0, stack, false)) { // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer - string trans_num_str; - trans_num_str.printf("%" UQUADFORMAT, rpb->rpb_transaction_nr); - ERR_post(Arg::Gds(isc_deadlock) << Arg::Gds(isc_update_conflict) << - Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str)); + Arg::Gds(isc_concurrent_transaction) << Arg::Int64(rpb->rpb_transaction_nr)); } // Old record was restored and re-fetched for write. Now replace it. @@ -2617,9 +2609,7 @@ bool VIO_get_current(thread_db* tdbb, if (!(transaction->tra_flags & TRA_ignore_limbo)) { // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer - string trans_num_str; - trans_num_str.printf("%" UQUADFORMAT, rpb->rpb_transaction_nr); - ERR_post(Arg::Gds(isc_rec_in_limbo) << Arg::Str(trans_num_str)); + ERR_post(Arg::Gds(isc_rec_in_limbo) << Arg::Int64(rpb->rpb_transaction_nr)); } // fall thru @@ -3160,12 +3150,9 @@ void VIO_modify(thread_db* tdbb, record_param* org_rpb, record_param* new_rpb, j stack, false)) { // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer - string trans_num_str; - trans_num_str.printf("%" UQUADFORMAT, org_rpb->rpb_transaction_nr); - ERR_post(Arg::Gds(isc_deadlock) << Arg::Gds(isc_update_conflict) << - Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str)); + Arg::Gds(isc_concurrent_transaction) << Arg::Int64(org_rpb->rpb_transaction_nr)); } IDX_modify_flag_uk_modified(tdbb, org_rpb, new_rpb, transaction); @@ -3401,12 +3388,9 @@ bool VIO_refetch_record(thread_db* tdbb, record_param* rpb, jrd_tra* transaction tdbb->bumpRelStats(RuntimeStatistics::RECORD_CONFLICTS, rpb->rpb_relation->rel_id); // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer - string trans_num_str; - trans_num_str.printf("%" UQUADFORMAT, rpb->rpb_transaction_nr); - ERR_post(Arg::Gds(isc_deadlock) << Arg::Gds(isc_update_conflict) << - Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str)); + Arg::Gds(isc_concurrent_transaction) << Arg::Int64(rpb->rpb_transaction_nr)); } return true; @@ -4021,19 +4005,14 @@ bool VIO_writelock(thread_db* tdbb, record_param* org_rpb, jrd_tra* transaction) org_rpb->rpb_runtime_flags |= RPB_refetch; return false; case PREPARE_LOCKERR: - { - // We got some kind of locking error (deadlock, timeout or lock_conflict) - // Error details should be stuffed into status vector at this point - // hvlad: we have no details as TRA_wait has already cleared the status vector + // We got some kind of locking error (deadlock, timeout or lock_conflict) + // Error details should be stuffed into status vector at this point + // hvlad: we have no details as TRA_wait has already cleared the status vector - // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer - string trans_num_str; - trans_num_str.printf("%" UQUADFORMAT, org_rpb->rpb_transaction_nr); - - ERR_post(Arg::Gds(isc_deadlock) << - Arg::Gds(isc_update_conflict) << - Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str)); - } + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + ERR_post(Arg::Gds(isc_deadlock) << + Arg::Gds(isc_update_conflict) << + Arg::Gds(isc_concurrent_transaction) << Arg::Int64(org_rpb->rpb_transaction_nr)); } // Old record was restored and re-fetched for write. Now replace it. @@ -5728,20 +5707,15 @@ static int prepare_update( thread_db* tdbb, tdbb->bumpRelStats(RuntimeStatistics::RECORD_CONFLICTS, relation->rel_id); // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer - string trans_num_str; - trans_num_str.printf("%" UQUADFORMAT, update_conflict_trans); - ERR_post(Arg::Gds(isc_update_conflict) << - Arg::Gds(isc_concurrent_transaction) << Arg::Str(trans_num_str)); + Arg::Gds(isc_concurrent_transaction) << Arg::Int64(update_conflict_trans)); } case tra_limbo: if (!(transaction->tra_flags & TRA_ignore_limbo)) { // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer - string trans_num_str; - trans_num_str.printf("%" UQUADFORMAT, rpb->rpb_transaction_nr); - ERR_post(Arg::Gds(isc_rec_in_limbo) << Arg::Str(trans_num_str)); + ERR_post(Arg::Gds(isc_rec_in_limbo) << Arg::Int64(rpb->rpb_transaction_nr)); } // fall thru From 1d350746d7a865e8f5e408a08a40d33f40d09cc4 Mon Sep 17 00:00:00 2001 From: hvlad Date: Wed, 12 Feb 2020 15:48:21 +0200 Subject: [PATCH 273/274] Correct usage of Arg::Int64 --- src/common/StatusArg.h | 4 +++- src/dsql/DdlNodes.epp | 12 ++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/common/StatusArg.h b/src/common/StatusArg.h index 9540240c1a..ee43baa8ba 100644 --- a/src/common/StatusArg.h +++ b/src/common/StatusArg.h @@ -249,7 +249,9 @@ public: }; // On 32-bit architecture ISC_STATUS can't fit 64-bit integer therefore -// convert such a numbers into text and put string into status-vector +// convert such a numbers into text and put string into status-vector. +// Make sure that temporary instance of this class is not going out of scope +// before exception is raised ! class Int64 : public Str { public: diff --git a/src/dsql/DdlNodes.epp b/src/dsql/DdlNodes.epp index 1d8ebeb8d8..4586aa97b5 100644 --- a/src/dsql/DdlNodes.epp +++ b/src/dsql/DdlNodes.epp @@ -12426,15 +12426,11 @@ void AlterDatabaseNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratc delete lock; // msg 297: Concurrent ALTER DATABASE is not supported - Arg::PrivateDyn status(297); + if (!conflict_trans) + Arg::PrivateDyn(297).raise(); - if (conflict_trans) - { - // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer - status << Arg::Gds(isc_concurrent_transaction) << Arg::Int64(conflict_trans); - } - - status_exception::raise(status); + // Cannot use Arg::Num here because transaction number is 64-bit unsigned integer + (Arg::PrivateDyn(297) << Arg::Gds(isc_concurrent_transaction) << Arg::Int64(conflict_trans)).raise(); } } From 19713e76157605219e84a230c051bb2bf6e6262f Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Wed, 12 Feb 2020 12:12:28 -0300 Subject: [PATCH 274/274] Unzip tzdata with unzip utility in Windows. --- .github/workflows/main.yml | 2 - builds/win32/make_icu.bat | 2 +- builds/win32/zipjs.bat | 823 ------------------------------------- 3 files changed, 1 insertion(+), 826 deletions(-) delete mode 100644 builds/win32/zipjs.bat diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3d7d1490fe..cd0549b67c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -110,14 +110,12 @@ jobs: env: PLATFORM: ${{ matrix.platform }} run: | - set if "%PLATFORM%" == "x64" set FB_VS_ARCH=amd64 if "%PLATFORM%" == "x64" set FB_PROCESSOR_ARCHITECTURE=AMD64 if "%PLATFORM%" == "x64" set FB_OUTPUT_SUFFIX=x64 if "%PLATFORM%" == "x86" set FB_VS_ARCH=x86 if "%PLATFORM%" == "x86" set FB_PROCESSOR_ARCHITECTURE=x86 if "%PLATFORM%" == "x86" set FB_OUTPUT_SUFFIX=win32 - set call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=%FB_VS_ARCH% cd builds\win32 run_all.bat JUSTBUILD diff --git a/builds/win32/make_icu.bat b/builds/win32/make_icu.bat index ced0139464..2f9f610b7d 100644 --- a/builds/win32/make_icu.bat +++ b/builds/win32/make_icu.bat @@ -15,7 +15,7 @@ if errorlevel 1 call :ERROR build failed - see make_icu_%FB_TARGET_PLATFORM%.log @echo Extracting tzdata mkdir %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\tzdata -call zipjs.bat unzip -source "%FB_LONG_ROOT_PATH%\extern\icu\tzdata\le.zip" -destination %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\tzdata -keep yes +unzip -o %FB_ROOT_PATH%\extern\icu\tzdata\le.zip -d %FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\firebird\tzdata @goto :EOF diff --git a/builds/win32/zipjs.bat b/builds/win32/zipjs.bat deleted file mode 100644 index 02b8fcbc24..0000000000 --- a/builds/win32/zipjs.bat +++ /dev/null @@ -1,823 +0,0 @@ -@if (@X)==(@Y) @end /* JScript comment - @echo off - - rem :: the first argument is the script name as it will be used for proper help message - cscript //E:JScript //nologo "%~f0" "%~nx0" %* - - exit /b %errorlevel% - -@if (@X)==(@Y) @end JScript comment */ - - -/* -Compression/uncompression command-line tool that uses Shell.Application and WSH/Jscript - -http://msdn.microsoft.com/en-us/library/windows/desktop/bb774085(v=vs.85).aspx - -Some resources That I've used: -http://www.robvanderwoude.com/vbstech_files_zip.php -https://code.google.com/p/jsxt/source/browse/trunk/js/win32/ZipFile.js?r=161 - - - - -UPDATE *17-03-15* - -Devnullius Plussed noticed a bug in ZipDirItems and ZipItem functions (now fixed) -And also following issues (at the moment not handled by the script): -- if there's not enough space on the system drive (usually C:\) the script could produce various errors , most often the script halts. -- Folders and files that contain unicode symbols cannot be handled by Shell.Application object. - -UPDATE *24-03-15* - -Error messages are caught in waitforcount method and if shuch pops-up the script is stopped. -As I don't know hoe to check the content of the pop-up the exact reason for the failure is not given -but only the possible reasons. - -UPDATE *22-02-16* - -Javid Pack(https://github.com/JavidPack) has found two bugs in zipItem command and in ZipItem function.Now fixed. - ------- -It's possible to be ported for C#,Powershell and JScript.net so I'm planning to do it at some time. - -For sure there's a lot of room for improvements and optimization and I'm absolutely sure there are some bugs -as the script is big enough to not have. - - - -!!! -For suggestions contact me at - npocmaka@gmail.com -!!! - -*/ - - -////////////////////////////////////// -// CONSTANTS - -// TODO - Shell.Application and Scripting.FileSystemObject objects could be set as global variables to avoid theit creation -// in every method. - -//empty zip character sequense -var ZIP_DATA= "PK" + String.fromCharCode(5) + String.fromCharCode(6) + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; - -var SLEEP_INTERVAL=200; - -//copy option(s) used by Shell.Application.CopyHere/MoveHere -var NO_PROGRESS_BAR=4; - - -//oprions used for zip/unzip -var force=true; -var move=false; - -//option used for listing content of archive -var flat=false; - -var source=""; -var destination=""; - -var ARGS = WScript.Arguments; -var scriptName=ARGS.Item(0); - -// -////////////////////////////////////// - -////////////////////////////////////// -// ADODB.Stream extensions - -if ( ! this.ADODB ) { - var ADODB = {}; -} - -if ( ! ADODB.Stream ) { - ADODB.Stream = {}; -} - -// writes a binary data to a file -if ( ! ADODB.Stream.writeFile ) { - ADODB.Stream.writeFile = function(filename, bindata) - { - var stream = new ActiveXObject("ADODB.Stream"); - stream.Type = 2; - stream.Mode = 3; - stream.Charset ="ASCII"; - stream.Open(); - stream.Position = 0; - stream.WriteText(bindata); - stream.SaveToFile(filename, 2); - stream.Close(); - return true; - }; -} - -// -////////////////////////////////////// - -////////////////////////////////////// -// common - -if ( ! this.Common ) { - var Common = {}; -} - -if ( ! Common.WaitForCount ) { - Common.WaitForCount = function(folderObject,targetCount,countFunction){ - var shell = new ActiveXObject("Wscript.Shell"); - while (countFunction(folderObject) < targetCount ){ - WScript.Sleep(SLEEP_INTERVAL); - //checks if a pop-up with error message appears while zipping - //at the moment I have no idea how to read the pop-up content - // to give the exact reason for failing - if (shell.AppActivate("Compressed (zipped) Folders Error")) { - WScript.Echo("Error While zipping"); - WScript.Echo(""); - WScript.Echo("Possible reasons:"); - WScript.Echo(" -source contains filename(s) with unicode characters"); - WScript.Echo(" -produces zip exceeds 8gb size (or 2,5 gb for XP and 2003)"); - WScript.Echo(" -not enough space on system drive (usually C:\\)"); - WScript.Quit(432); - } - - } - } -} - -if ( ! Common.getParent ) { - Common.getParent = function(path){ - var splitted=path.split("\\"); - var result=""; - for (var s=0;s