| /* | |
| ******************************************************************************* | |
| * Copyright (C) 2013-2014, International Business Machines | |
| * Corporation and others. All Rights Reserved. | |
| ******************************************************************************* | |
| * collationbuilder.cpp | |
| * | |
| * (replaced the former ucol_bld.cpp) | |
| * | |
| * created on: 2013may06 | |
| * created by: Markus W. Scherer | |
| */ | |
| #ifdef DEBUG_COLLATION_BUILDER | |
| #if !defined(STARBOARD) | |
| #include <stdio.h> | |
| #endif | |
| #endif | |
| #include "unicode/utypes.h" | |
| #if !UCONFIG_NO_COLLATION | |
| #include "starboard/client_porting/poem/assert_poem.h" | |
| #include "starboard/client_porting/poem/string_poem.h" | |
| #include "unicode/caniter.h" | |
| #include "unicode/normalizer2.h" | |
| #include "unicode/tblcoll.h" | |
| #include "unicode/parseerr.h" | |
| #include "unicode/uchar.h" | |
| #include "unicode/ucol.h" | |
| #include "unicode/unistr.h" | |
| #include "unicode/usetiter.h" | |
| #include "unicode/utf16.h" | |
| #include "unicode/uversion.h" | |
| #include "cmemory.h" | |
| #include "collation.h" | |
| #include "collationbuilder.h" | |
| #include "collationdata.h" | |
| #include "collationdatabuilder.h" | |
| #include "collationfastlatin.h" | |
| #include "collationroot.h" | |
| #include "collationrootelements.h" | |
| #include "collationruleparser.h" | |
| #include "collationsettings.h" | |
| #include "collationtailoring.h" | |
| #include "collationweights.h" | |
| #include "normalizer2impl.h" | |
| #include "uassert.h" | |
| #include "ucol_imp.h" | |
| #include "utf16collationiterator.h" | |
| U_NAMESPACE_BEGIN | |
| namespace { | |
| class BundleImporter : public CollationRuleParser::Importer { | |
| public: | |
| BundleImporter() {} | |
| virtual ~BundleImporter(); | |
| virtual void getRules( | |
| const char *localeID, const char *collationType, | |
| UnicodeString &rules, | |
| const char *&errorReason, UErrorCode &errorCode); | |
| }; | |
| BundleImporter::~BundleImporter() {} | |
| void | |
| BundleImporter::getRules( | |
| const char *localeID, const char *collationType, | |
| UnicodeString &rules, | |
| const char *& /*errorReason*/, UErrorCode &errorCode) { | |
| CollationLoader::loadRules(localeID, collationType, rules, errorCode); | |
| } | |
| } // namespace | |
| // RuleBasedCollator implementation ---------------------------------------- *** | |
| // These methods are here, rather than in rulebasedcollator.cpp, | |
| // for modularization: | |
| // Most code using Collator does not need to build a Collator from rules. | |
| // By moving these constructors and helper methods to a separate file, | |
| // most code will not have a static dependency on the builder code. | |
| RuleBasedCollator::RuleBasedCollator() | |
| : data(NULL), | |
| settings(NULL), | |
| tailoring(NULL), | |
| cacheEntry(NULL), | |
| validLocale(""), | |
| explicitlySetAttributes(0), | |
| actualLocaleIsSameAsValid(FALSE) { | |
| } | |
| RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, UErrorCode &errorCode) | |
| : data(NULL), | |
| settings(NULL), | |
| tailoring(NULL), | |
| cacheEntry(NULL), | |
| validLocale(""), | |
| explicitlySetAttributes(0), | |
| actualLocaleIsSameAsValid(FALSE) { | |
| internalBuildTailoring(rules, UCOL_DEFAULT, UCOL_DEFAULT, NULL, NULL, errorCode); | |
| } | |
| RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, ECollationStrength strength, | |
| UErrorCode &errorCode) | |
| : data(NULL), | |
| settings(NULL), | |
| tailoring(NULL), | |
| cacheEntry(NULL), | |
| validLocale(""), | |
| explicitlySetAttributes(0), | |
| actualLocaleIsSameAsValid(FALSE) { | |
| internalBuildTailoring(rules, strength, UCOL_DEFAULT, NULL, NULL, errorCode); | |
| } | |
| RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, | |
| UColAttributeValue decompositionMode, | |
| UErrorCode &errorCode) | |
| : data(NULL), | |
| settings(NULL), | |
| tailoring(NULL), | |
| cacheEntry(NULL), | |
| validLocale(""), | |
| explicitlySetAttributes(0), | |
| actualLocaleIsSameAsValid(FALSE) { | |
| internalBuildTailoring(rules, UCOL_DEFAULT, decompositionMode, NULL, NULL, errorCode); | |
| } | |
| RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, | |
| ECollationStrength strength, | |
| UColAttributeValue decompositionMode, | |
| UErrorCode &errorCode) | |
| : data(NULL), | |
| settings(NULL), | |
| tailoring(NULL), | |
| cacheEntry(NULL), | |
| validLocale(""), | |
| explicitlySetAttributes(0), | |
| actualLocaleIsSameAsValid(FALSE) { | |
| internalBuildTailoring(rules, strength, decompositionMode, NULL, NULL, errorCode); | |
| } | |
| RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, | |
| UParseError &parseError, UnicodeString &reason, | |
| UErrorCode &errorCode) | |
| : data(NULL), | |
| settings(NULL), | |
| tailoring(NULL), | |
| cacheEntry(NULL), | |
| validLocale(""), | |
| explicitlySetAttributes(0), | |
| actualLocaleIsSameAsValid(FALSE) { | |
| internalBuildTailoring(rules, UCOL_DEFAULT, UCOL_DEFAULT, &parseError, &reason, errorCode); | |
| } | |
| void | |
| RuleBasedCollator::internalBuildTailoring(const UnicodeString &rules, | |
| int32_t strength, | |
| UColAttributeValue decompositionMode, | |
| UParseError *outParseError, UnicodeString *outReason, | |
| UErrorCode &errorCode) { | |
| const CollationTailoring *base = CollationRoot::getRoot(errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| if(outReason != NULL) { outReason->remove(); } | |
| CollationBuilder builder(base, errorCode); | |
| UVersionInfo noVersion = { 0, 0, 0, 0 }; | |
| BundleImporter importer; | |
| LocalPointer<CollationTailoring> t(builder.parseAndBuild(rules, noVersion, | |
| &importer, | |
| outParseError, errorCode)); | |
| if(U_FAILURE(errorCode)) { | |
| const char *reason = builder.getErrorReason(); | |
| if(reason != NULL && outReason != NULL) { | |
| *outReason = UnicodeString(reason, -1, US_INV); | |
| } | |
| return; | |
| } | |
| t->actualLocale.setToBogus(); | |
| adoptTailoring(t.orphan(), errorCode); | |
| // Set attributes after building the collator, | |
| // to keep the default settings consistent with the rule string. | |
| if(strength != UCOL_DEFAULT) { | |
| setAttribute(UCOL_STRENGTH, (UColAttributeValue)strength, errorCode); | |
| } | |
| if(decompositionMode != UCOL_DEFAULT) { | |
| setAttribute(UCOL_NORMALIZATION_MODE, decompositionMode, errorCode); | |
| } | |
| } | |
| // CollationBuilder implementation ----------------------------------------- *** | |
| // Some compilers don't care if constants are defined in the .cpp file. | |
| // MS Visual C++ does not like it, but gcc requires it. clang does not care. | |
| #ifndef _MSC_VER | |
| const int32_t CollationBuilder::HAS_BEFORE2; | |
| const int32_t CollationBuilder::HAS_BEFORE3; | |
| #endif | |
| CollationBuilder::CollationBuilder(const CollationTailoring *b, UErrorCode &errorCode) | |
| : nfd(*Normalizer2::getNFDInstance(errorCode)), | |
| fcd(*Normalizer2Factory::getFCDInstance(errorCode)), | |
| nfcImpl(*Normalizer2Factory::getNFCImpl(errorCode)), | |
| base(b), | |
| baseData(b->data), | |
| rootElements(b->data->rootElements, b->data->rootElementsLength), | |
| variableTop(0), | |
| dataBuilder(new CollationDataBuilder(errorCode)), fastLatinEnabled(TRUE), | |
| errorReason(NULL), | |
| cesLength(0), | |
| rootPrimaryIndexes(errorCode), nodes(errorCode) { | |
| nfcImpl.ensureCanonIterData(errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| errorReason = "CollationBuilder fields initialization failed"; | |
| return; | |
| } | |
| if(dataBuilder == NULL) { | |
| errorCode = U_MEMORY_ALLOCATION_ERROR; | |
| return; | |
| } | |
| dataBuilder->initForTailoring(baseData, errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| errorReason = "CollationBuilder initialization failed"; | |
| } | |
| } | |
| CollationBuilder::~CollationBuilder() { | |
| delete dataBuilder; | |
| } | |
| CollationTailoring * | |
| CollationBuilder::parseAndBuild(const UnicodeString &ruleString, | |
| const UVersionInfo rulesVersion, | |
| CollationRuleParser::Importer *importer, | |
| UParseError *outParseError, | |
| UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return NULL; } | |
| if(baseData->rootElements == NULL) { | |
| errorCode = U_MISSING_RESOURCE_ERROR; | |
| errorReason = "missing root elements data, tailoring not supported"; | |
| return NULL; | |
| } | |
| LocalPointer<CollationTailoring> tailoring(new CollationTailoring(base->settings)); | |
| if(tailoring.isNull() || tailoring->isBogus()) { | |
| errorCode = U_MEMORY_ALLOCATION_ERROR; | |
| return NULL; | |
| } | |
| CollationRuleParser parser(baseData, errorCode); | |
| if(U_FAILURE(errorCode)) { return NULL; } | |
| // Note: This always bases &[last variable] and &[first regular] | |
| // on the root collator's maxVariable/variableTop. | |
| // If we wanted this to change after [maxVariable x], then we would keep | |
| // the tailoring.settings pointer here and read its variableTop when we need it. | |
| // See http://unicode.org/cldr/trac/ticket/6070 | |
| variableTop = base->settings->variableTop; | |
| parser.setSink(this); | |
| parser.setImporter(importer); | |
| CollationSettings &ownedSettings = *SharedObject::copyOnWrite(tailoring->settings); | |
| parser.parse(ruleString, ownedSettings, outParseError, errorCode); | |
| errorReason = parser.getErrorReason(); | |
| if(U_FAILURE(errorCode)) { return NULL; } | |
| if(dataBuilder->hasMappings()) { | |
| makeTailoredCEs(errorCode); | |
| closeOverComposites(errorCode); | |
| finalizeCEs(errorCode); | |
| // Copy all of ASCII, and Latin-1 letters, into each tailoring. | |
| optimizeSet.add(0, 0x7f); | |
| optimizeSet.add(0xc0, 0xff); | |
| // Hangul is decomposed on the fly during collation, | |
| // and the tailoring data is always built with HANGUL_TAG specials. | |
| optimizeSet.remove(Hangul::HANGUL_BASE, Hangul::HANGUL_END); | |
| dataBuilder->optimize(optimizeSet, errorCode); | |
| tailoring->ensureOwnedData(errorCode); | |
| if(U_FAILURE(errorCode)) { return NULL; } | |
| if(fastLatinEnabled) { dataBuilder->enableFastLatin(); } | |
| dataBuilder->build(*tailoring->ownedData, errorCode); | |
| tailoring->builder = dataBuilder; | |
| dataBuilder = NULL; | |
| } else { | |
| tailoring->data = baseData; | |
| } | |
| if(U_FAILURE(errorCode)) { return NULL; } | |
| ownedSettings.fastLatinOptions = CollationFastLatin::getOptions( | |
| tailoring->data, ownedSettings, | |
| ownedSettings.fastLatinPrimaries, UPRV_LENGTHOF(ownedSettings.fastLatinPrimaries)); | |
| tailoring->rules = ruleString; | |
| tailoring->rules.getTerminatedBuffer(); // ensure NUL-termination | |
| tailoring->setVersion(base->version, rulesVersion); | |
| return tailoring.orphan(); | |
| } | |
| void | |
| CollationBuilder::addReset(int32_t strength, const UnicodeString &str, | |
| const char *&parserErrorReason, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| U_ASSERT(!str.isEmpty()); | |
| if(str.charAt(0) == CollationRuleParser::POS_LEAD) { | |
| ces[0] = getSpecialResetPosition(str, parserErrorReason, errorCode); | |
| cesLength = 1; | |
| if(U_FAILURE(errorCode)) { return; } | |
| U_ASSERT((ces[0] & Collation::CASE_AND_QUATERNARY_MASK) == 0); | |
| } else { | |
| // normal reset to a character or string | |
| UnicodeString nfdString = nfd.normalize(str, errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| parserErrorReason = "normalizing the reset position"; | |
| return; | |
| } | |
| cesLength = dataBuilder->getCEs(nfdString, ces, 0); | |
| if(cesLength > Collation::MAX_EXPANSION_LENGTH) { | |
| errorCode = U_ILLEGAL_ARGUMENT_ERROR; | |
| parserErrorReason = "reset position maps to too many collation elements (more than 31)"; | |
| return; | |
| } | |
| } | |
| if(strength == UCOL_IDENTICAL) { return; } // simple reset-at-position | |
| // &[before strength]position | |
| U_ASSERT(UCOL_PRIMARY <= strength && strength <= UCOL_TERTIARY); | |
| int32_t index = findOrInsertNodeForCEs(strength, parserErrorReason, errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| int64_t node = nodes.elementAti(index); | |
| // If the index is for a "weaker" node, | |
| // then skip backwards over this and further "weaker" nodes. | |
| while(strengthFromNode(node) > strength) { | |
| index = previousIndexFromNode(node); | |
| node = nodes.elementAti(index); | |
| } | |
| // Find or insert a node whose index we will put into a temporary CE. | |
| if(strengthFromNode(node) == strength && isTailoredNode(node)) { | |
| // Reset to just before this same-strength tailored node. | |
| index = previousIndexFromNode(node); | |
| } else if(strength == UCOL_PRIMARY) { | |
| // root primary node (has no previous index) | |
| uint32_t p = weight32FromNode(node); | |
| if(p == 0) { | |
| errorCode = U_UNSUPPORTED_ERROR; | |
| parserErrorReason = "reset primary-before ignorable not possible"; | |
| return; | |
| } | |
| if(p <= rootElements.getFirstPrimary()) { | |
| // There is no primary gap between ignorables and the space-first-primary. | |
| errorCode = U_UNSUPPORTED_ERROR; | |
| parserErrorReason = "reset primary-before first non-ignorable not supported"; | |
| return; | |
| } | |
| if(p == Collation::FIRST_TRAILING_PRIMARY) { | |
| // We do not support tailoring to an unassigned-implicit CE. | |
| errorCode = U_UNSUPPORTED_ERROR; | |
| parserErrorReason = "reset primary-before [first trailing] not supported"; | |
| return; | |
| } | |
| p = rootElements.getPrimaryBefore(p, baseData->isCompressiblePrimary(p)); | |
| index = findOrInsertNodeForPrimary(p, errorCode); | |
| // Go to the last node in this list: | |
| // Tailor after the last node between adjacent root nodes. | |
| for(;;) { | |
| node = nodes.elementAti(index); | |
| int32_t nextIndex = nextIndexFromNode(node); | |
| if(nextIndex == 0) { break; } | |
| index = nextIndex; | |
| } | |
| } else { | |
| // &[before 2] or &[before 3] | |
| index = findCommonNode(index, UCOL_SECONDARY); | |
| if(strength >= UCOL_TERTIARY) { | |
| index = findCommonNode(index, UCOL_TERTIARY); | |
| } | |
| // findCommonNode() stayed on the stronger node or moved to | |
| // an explicit common-weight node of the reset-before strength. | |
| node = nodes.elementAti(index); | |
| if(strengthFromNode(node) == strength) { | |
| // Found a same-strength node with an explicit weight. | |
| uint32_t weight16 = weight16FromNode(node); | |
| if(weight16 == 0) { | |
| errorCode = U_UNSUPPORTED_ERROR; | |
| if(strength == UCOL_SECONDARY) { | |
| parserErrorReason = "reset secondary-before secondary ignorable not possible"; | |
| } else { | |
| parserErrorReason = "reset tertiary-before completely ignorable not possible"; | |
| } | |
| return; | |
| } | |
| U_ASSERT(weight16 > Collation::BEFORE_WEIGHT16); | |
| // Reset to just before this node. | |
| // Insert the preceding same-level explicit weight if it is not there already. | |
| // Which explicit weight immediately precedes this one? | |
| weight16 = getWeight16Before(index, node, strength); | |
| // Does this preceding weight have a node? | |
| uint32_t previousWeight16; | |
| int32_t previousIndex = previousIndexFromNode(node); | |
| for(int32_t i = previousIndex;; i = previousIndexFromNode(node)) { | |
| node = nodes.elementAti(i); | |
| int32_t previousStrength = strengthFromNode(node); | |
| if(previousStrength < strength) { | |
| U_ASSERT(weight16 >= Collation::COMMON_WEIGHT16 || i == previousIndex); | |
| // Either the reset element has an above-common weight and | |
| // the parent node provides the implied common weight, | |
| // or the reset element has a weight<=common in the node | |
| // right after the parent, and we need to insert the preceding weight. | |
| previousWeight16 = Collation::COMMON_WEIGHT16; | |
| break; | |
| } else if(previousStrength == strength && !isTailoredNode(node)) { | |
| previousWeight16 = weight16FromNode(node); | |
| break; | |
| } | |
| // Skip weaker nodes and same-level tailored nodes. | |
| } | |
| if(previousWeight16 == weight16) { | |
| // The preceding weight has a node, | |
| // maybe with following weaker or tailored nodes. | |
| // Reset to the last of them. | |
| index = previousIndex; | |
| } else { | |
| // Insert a node with the preceding weight, reset to that. | |
| node = nodeFromWeight16(weight16) | nodeFromStrength(strength); | |
| index = insertNodeBetween(previousIndex, index, node, errorCode); | |
| } | |
| } else { | |
| // Found a stronger node with implied strength-common weight. | |
| uint32_t weight16 = getWeight16Before(index, node, strength); | |
| index = findOrInsertWeakNode(index, weight16, strength, errorCode); | |
| } | |
| // Strength of the temporary CE = strength of its reset position. | |
| // Code above raises an error if the before-strength is stronger. | |
| strength = ceStrength(ces[cesLength - 1]); | |
| } | |
| if(U_FAILURE(errorCode)) { | |
| parserErrorReason = "inserting reset position for &[before n]"; | |
| return; | |
| } | |
| ces[cesLength - 1] = tempCEFromIndexAndStrength(index, strength); | |
| } | |
| uint32_t | |
| CollationBuilder::getWeight16Before(int32_t index, int64_t node, int32_t level) { | |
| U_ASSERT(strengthFromNode(node) < level || !isTailoredNode(node)); | |
| // Collect the root CE weights if this node is for a root CE. | |
| // If it is not, then return the low non-primary boundary for a tailored CE. | |
| uint32_t t; | |
| if(strengthFromNode(node) == UCOL_TERTIARY) { | |
| t = weight16FromNode(node); | |
| } else { | |
| t = Collation::COMMON_WEIGHT16; // Stronger node with implied common weight. | |
| } | |
| while(strengthFromNode(node) > UCOL_SECONDARY) { | |
| index = previousIndexFromNode(node); | |
| node = nodes.elementAti(index); | |
| } | |
| if(isTailoredNode(node)) { | |
| return Collation::BEFORE_WEIGHT16; | |
| } | |
| uint32_t s; | |
| if(strengthFromNode(node) == UCOL_SECONDARY) { | |
| s = weight16FromNode(node); | |
| } else { | |
| s = Collation::COMMON_WEIGHT16; // Stronger node with implied common weight. | |
| } | |
| while(strengthFromNode(node) > UCOL_PRIMARY) { | |
| index = previousIndexFromNode(node); | |
| node = nodes.elementAti(index); | |
| } | |
| if(isTailoredNode(node)) { | |
| return Collation::BEFORE_WEIGHT16; | |
| } | |
| // [p, s, t] is a root CE. Return the preceding weight for the requested level. | |
| uint32_t p = weight32FromNode(node); | |
| uint32_t weight16; | |
| if(level == UCOL_SECONDARY) { | |
| weight16 = rootElements.getSecondaryBefore(p, s); | |
| } else { | |
| weight16 = rootElements.getTertiaryBefore(p, s, t); | |
| U_ASSERT((weight16 & ~Collation::ONLY_TERTIARY_MASK) == 0); | |
| } | |
| return weight16; | |
| } | |
| int64_t | |
| CollationBuilder::getSpecialResetPosition(const UnicodeString &str, | |
| const char *&parserErrorReason, UErrorCode &errorCode) { | |
| U_ASSERT(str.length() == 2); | |
| int64_t ce; | |
| int32_t strength = UCOL_PRIMARY; | |
| UBool isBoundary = FALSE; | |
| UChar32 pos = str.charAt(1) - CollationRuleParser::POS_BASE; | |
| U_ASSERT(0 <= pos && pos <= CollationRuleParser::LAST_TRAILING); | |
| switch(pos) { | |
| case CollationRuleParser::FIRST_TERTIARY_IGNORABLE: | |
| // Quaternary CEs are not supported. | |
| // Non-zero quaternary weights are possible only on tertiary or stronger CEs. | |
| return 0; | |
| case CollationRuleParser::LAST_TERTIARY_IGNORABLE: | |
| return 0; | |
| case CollationRuleParser::FIRST_SECONDARY_IGNORABLE: { | |
| // Look for a tailored tertiary node after [0, 0, 0]. | |
| int32_t index = findOrInsertNodeForRootCE(0, UCOL_TERTIARY, errorCode); | |
| if(U_FAILURE(errorCode)) { return 0; } | |
| int64_t node = nodes.elementAti(index); | |
| if((index = nextIndexFromNode(node)) != 0) { | |
| node = nodes.elementAti(index); | |
| U_ASSERT(strengthFromNode(node) <= UCOL_TERTIARY); | |
| if(isTailoredNode(node) && strengthFromNode(node) == UCOL_TERTIARY) { | |
| return tempCEFromIndexAndStrength(index, UCOL_TERTIARY); | |
| } | |
| } | |
| return rootElements.getFirstTertiaryCE(); | |
| // No need to look for nodeHasAnyBefore() on a tertiary node. | |
| } | |
| case CollationRuleParser::LAST_SECONDARY_IGNORABLE: | |
| ce = rootElements.getLastTertiaryCE(); | |
| strength = UCOL_TERTIARY; | |
| break; | |
| case CollationRuleParser::FIRST_PRIMARY_IGNORABLE: { | |
| // Look for a tailored secondary node after [0, 0, *]. | |
| int32_t index = findOrInsertNodeForRootCE(0, UCOL_SECONDARY, errorCode); | |
| if(U_FAILURE(errorCode)) { return 0; } | |
| int64_t node = nodes.elementAti(index); | |
| while((index = nextIndexFromNode(node)) != 0) { | |
| node = nodes.elementAti(index); | |
| strength = strengthFromNode(node); | |
| if(strength < UCOL_SECONDARY) { break; } | |
| if(strength == UCOL_SECONDARY) { | |
| if(isTailoredNode(node)) { | |
| if(nodeHasBefore3(node)) { | |
| index = nextIndexFromNode(nodes.elementAti(nextIndexFromNode(node))); | |
| U_ASSERT(isTailoredNode(nodes.elementAti(index))); | |
| } | |
| return tempCEFromIndexAndStrength(index, UCOL_SECONDARY); | |
| } else { | |
| break; | |
| } | |
| } | |
| } | |
| ce = rootElements.getFirstSecondaryCE(); | |
| strength = UCOL_SECONDARY; | |
| break; | |
| } | |
| case CollationRuleParser::LAST_PRIMARY_IGNORABLE: | |
| ce = rootElements.getLastSecondaryCE(); | |
| strength = UCOL_SECONDARY; | |
| break; | |
| case CollationRuleParser::FIRST_VARIABLE: | |
| ce = rootElements.getFirstPrimaryCE(); | |
| isBoundary = TRUE; // FractionalUCA.txt: FDD1 00A0, SPACE first primary | |
| break; | |
| case CollationRuleParser::LAST_VARIABLE: | |
| ce = rootElements.lastCEWithPrimaryBefore(variableTop + 1); | |
| break; | |
| case CollationRuleParser::FIRST_REGULAR: | |
| ce = rootElements.firstCEWithPrimaryAtLeast(variableTop + 1); | |
| isBoundary = TRUE; // FractionalUCA.txt: FDD1 263A, SYMBOL first primary | |
| break; | |
| case CollationRuleParser::LAST_REGULAR: | |
| // Use the Hani-first-primary rather than the actual last "regular" CE before it, | |
| // for backward compatibility with behavior before the introduction of | |
| // script-first-primary CEs in the root collator. | |
| ce = rootElements.firstCEWithPrimaryAtLeast( | |
| baseData->getFirstPrimaryForGroup(USCRIPT_HAN)); | |
| break; | |
| case CollationRuleParser::FIRST_IMPLICIT: | |
| ce = baseData->getSingleCE(0x4e00, errorCode); | |
| break; | |
| case CollationRuleParser::LAST_IMPLICIT: | |
| // We do not support tailoring to an unassigned-implicit CE. | |
| errorCode = U_UNSUPPORTED_ERROR; | |
| parserErrorReason = "reset to [last implicit] not supported"; | |
| return 0; | |
| case CollationRuleParser::FIRST_TRAILING: | |
| ce = Collation::makeCE(Collation::FIRST_TRAILING_PRIMARY); | |
| isBoundary = TRUE; // trailing first primary (there is no mapping for it) | |
| break; | |
| case CollationRuleParser::LAST_TRAILING: | |
| errorCode = U_ILLEGAL_ARGUMENT_ERROR; | |
| parserErrorReason = "LDML forbids tailoring to U+FFFF"; | |
| return 0; | |
| default: | |
| U_ASSERT(FALSE); | |
| return 0; | |
| } | |
| int32_t index = findOrInsertNodeForRootCE(ce, strength, errorCode); | |
| if(U_FAILURE(errorCode)) { return 0; } | |
| int64_t node = nodes.elementAti(index); | |
| if((pos & 1) == 0) { | |
| // even pos = [first xyz] | |
| if(!nodeHasAnyBefore(node) && isBoundary) { | |
| // A <group> first primary boundary is artificially added to FractionalUCA.txt. | |
| // It is reachable via its special contraction, but is not normally used. | |
| // Find the first character tailored after the boundary CE, | |
| // or the first real root CE after it. | |
| if((index = nextIndexFromNode(node)) != 0) { | |
| // If there is a following node, then it must be tailored | |
| // because there are no root CEs with a boundary primary | |
| // and non-common secondary/tertiary weights. | |
| node = nodes.elementAti(index); | |
| U_ASSERT(isTailoredNode(node)); | |
| ce = tempCEFromIndexAndStrength(index, strength); | |
| } else { | |
| U_ASSERT(strength == UCOL_PRIMARY); | |
| uint32_t p = (uint32_t)(ce >> 32); | |
| int32_t pIndex = rootElements.findPrimary(p); | |
| UBool isCompressible = baseData->isCompressiblePrimary(p); | |
| p = rootElements.getPrimaryAfter(p, pIndex, isCompressible); | |
| ce = Collation::makeCE(p); | |
| index = findOrInsertNodeForRootCE(ce, UCOL_PRIMARY, errorCode); | |
| if(U_FAILURE(errorCode)) { return 0; } | |
| node = nodes.elementAti(index); | |
| } | |
| } | |
| if(nodeHasAnyBefore(node)) { | |
| // Get the first node that was tailored before this one at a weaker strength. | |
| if(nodeHasBefore2(node)) { | |
| index = nextIndexFromNode(nodes.elementAti(nextIndexFromNode(node))); | |
| node = nodes.elementAti(index); | |
| } | |
| if(nodeHasBefore3(node)) { | |
| index = nextIndexFromNode(nodes.elementAti(nextIndexFromNode(node))); | |
| } | |
| U_ASSERT(isTailoredNode(nodes.elementAti(index))); | |
| ce = tempCEFromIndexAndStrength(index, strength); | |
| } | |
| } else { | |
| // odd pos = [last xyz] | |
| // Find the last node that was tailored after the [last xyz] | |
| // at a strength no greater than the position's strength. | |
| for(;;) { | |
| int32_t nextIndex = nextIndexFromNode(node); | |
| if(nextIndex == 0) { break; } | |
| int64_t nextNode = nodes.elementAti(nextIndex); | |
| if(strengthFromNode(nextNode) < strength) { break; } | |
| index = nextIndex; | |
| node = nextNode; | |
| } | |
| // Do not make a temporary CE for a root node. | |
| // This last node might be the node for the root CE itself, | |
| // or a node with a common secondary or tertiary weight. | |
| if(isTailoredNode(node)) { | |
| ce = tempCEFromIndexAndStrength(index, strength); | |
| } | |
| } | |
| return ce; | |
| } | |
| void | |
| CollationBuilder::addRelation(int32_t strength, const UnicodeString &prefix, | |
| const UnicodeString &str, const UnicodeString &extension, | |
| const char *&parserErrorReason, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| UnicodeString nfdPrefix; | |
| if(!prefix.isEmpty()) { | |
| nfd.normalize(prefix, nfdPrefix, errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| parserErrorReason = "normalizing the relation prefix"; | |
| return; | |
| } | |
| } | |
| UnicodeString nfdString = nfd.normalize(str, errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| parserErrorReason = "normalizing the relation string"; | |
| return; | |
| } | |
| // The runtime code decomposes Hangul syllables on the fly, | |
| // with recursive processing but without making the Jamo pieces visible for matching. | |
| // It does not work with certain types of contextual mappings. | |
| int32_t nfdLength = nfdString.length(); | |
| if(nfdLength >= 2) { | |
| UChar c = nfdString.charAt(0); | |
| if(Hangul::isJamoL(c) || Hangul::isJamoV(c)) { | |
| // While handling a Hangul syllable, contractions starting with Jamo L or V | |
| // would not see the following Jamo of that syllable. | |
| errorCode = U_UNSUPPORTED_ERROR; | |
| parserErrorReason = "contractions starting with conjoining Jamo L or V not supported"; | |
| return; | |
| } | |
| c = nfdString.charAt(nfdLength - 1); | |
| if(Hangul::isJamoL(c) || | |
| (Hangul::isJamoV(c) && Hangul::isJamoL(nfdString.charAt(nfdLength - 2)))) { | |
| // A contraction ending with Jamo L or L+V would require | |
| // generating Hangul syllables in addTailComposites() (588 for a Jamo L), | |
| // or decomposing a following Hangul syllable on the fly, during contraction matching. | |
| errorCode = U_UNSUPPORTED_ERROR; | |
| parserErrorReason = "contractions ending with conjoining Jamo L or L+V not supported"; | |
| return; | |
| } | |
| // A Hangul syllable completely inside a contraction is ok. | |
| } | |
| // Note: If there is a prefix, then the parser checked that | |
| // both the prefix and the string beging with NFC boundaries (not Jamo V or T). | |
| // Therefore: prefix.isEmpty() || !isJamoVOrT(nfdString.charAt(0)) | |
| // (While handling a Hangul syllable, prefixes on Jamo V or T | |
| // would not see the previous Jamo of that syllable.) | |
| if(strength != UCOL_IDENTICAL) { | |
| // Find the node index after which we insert the new tailored node. | |
| int32_t index = findOrInsertNodeForCEs(strength, parserErrorReason, errorCode); | |
| U_ASSERT(cesLength > 0); | |
| int64_t ce = ces[cesLength - 1]; | |
| if(strength == UCOL_PRIMARY && !isTempCE(ce) && (uint32_t)(ce >> 32) == 0) { | |
| // There is no primary gap between ignorables and the space-first-primary. | |
| errorCode = U_UNSUPPORTED_ERROR; | |
| parserErrorReason = "tailoring primary after ignorables not supported"; | |
| return; | |
| } | |
| if(strength == UCOL_QUATERNARY && ce == 0) { | |
| // The CE data structure does not support non-zero quaternary weights | |
| // on tertiary ignorables. | |
| errorCode = U_UNSUPPORTED_ERROR; | |
| parserErrorReason = "tailoring quaternary after tertiary ignorables not supported"; | |
| return; | |
| } | |
| // Insert the new tailored node. | |
| index = insertTailoredNodeAfter(index, strength, errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| parserErrorReason = "modifying collation elements"; | |
| return; | |
| } | |
| // Strength of the temporary CE: | |
| // The new relation may yield a stronger CE but not a weaker one. | |
| int32_t tempStrength = ceStrength(ce); | |
| if(strength < tempStrength) { tempStrength = strength; } | |
| ces[cesLength - 1] = tempCEFromIndexAndStrength(index, tempStrength); | |
| } | |
| setCaseBits(nfdString, parserErrorReason, errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| int32_t cesLengthBeforeExtension = cesLength; | |
| if(!extension.isEmpty()) { | |
| UnicodeString nfdExtension = nfd.normalize(extension, errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| parserErrorReason = "normalizing the relation extension"; | |
| return; | |
| } | |
| cesLength = dataBuilder->getCEs(nfdExtension, ces, cesLength); | |
| if(cesLength > Collation::MAX_EXPANSION_LENGTH) { | |
| errorCode = U_ILLEGAL_ARGUMENT_ERROR; | |
| parserErrorReason = | |
| "extension string adds too many collation elements (more than 31 total)"; | |
| return; | |
| } | |
| } | |
| uint32_t ce32 = Collation::UNASSIGNED_CE32; | |
| if((prefix != nfdPrefix || str != nfdString) && | |
| !ignorePrefix(prefix, errorCode) && !ignoreString(str, errorCode)) { | |
| // Map from the original input to the CEs. | |
| // We do this in case the canonical closure is incomplete, | |
| // so that it is possible to explicitly provide the missing mappings. | |
| ce32 = addIfDifferent(prefix, str, ces, cesLength, ce32, errorCode); | |
| } | |
| addWithClosure(nfdPrefix, nfdString, ces, cesLength, ce32, errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| parserErrorReason = "writing collation elements"; | |
| return; | |
| } | |
| cesLength = cesLengthBeforeExtension; | |
| } | |
| int32_t | |
| CollationBuilder::findOrInsertNodeForCEs(int32_t strength, const char *&parserErrorReason, | |
| UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return 0; } | |
| U_ASSERT(UCOL_PRIMARY <= strength && strength <= UCOL_QUATERNARY); | |
| // Find the last CE that is at least as "strong" as the requested difference. | |
| // Note: Stronger is smaller (UCOL_PRIMARY=0). | |
| int64_t ce; | |
| for(;; --cesLength) { | |
| if(cesLength == 0) { | |
| ce = ces[0] = 0; | |
| cesLength = 1; | |
| break; | |
| } else { | |
| ce = ces[cesLength - 1]; | |
| } | |
| if(ceStrength(ce) <= strength) { break; } | |
| } | |
| if(isTempCE(ce)) { | |
| // No need to findCommonNode() here for lower levels | |
| // because insertTailoredNodeAfter() will do that anyway. | |
| return indexFromTempCE(ce); | |
| } | |
| // root CE | |
| if((uint8_t)(ce >> 56) == Collation::UNASSIGNED_IMPLICIT_BYTE) { | |
| errorCode = U_UNSUPPORTED_ERROR; | |
| parserErrorReason = "tailoring relative to an unassigned code point not supported"; | |
| return 0; | |
| } | |
| return findOrInsertNodeForRootCE(ce, strength, errorCode); | |
| } | |
| int32_t | |
| CollationBuilder::findOrInsertNodeForRootCE(int64_t ce, int32_t strength, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return 0; } | |
| U_ASSERT((uint8_t)(ce >> 56) != Collation::UNASSIGNED_IMPLICIT_BYTE); | |
| // Find or insert the node for each of the root CE's weights, | |
| // down to the requested level/strength. | |
| // Root CEs must have common=zero quaternary weights (for which we never insert any nodes). | |
| U_ASSERT((ce & 0xc0) == 0); | |
| int32_t index = findOrInsertNodeForPrimary((uint32_t)(ce >> 32), errorCode); | |
| if(strength >= UCOL_SECONDARY) { | |
| uint32_t lower32 = (uint32_t)ce; | |
| index = findOrInsertWeakNode(index, lower32 >> 16, UCOL_SECONDARY, errorCode); | |
| if(strength >= UCOL_TERTIARY) { | |
| index = findOrInsertWeakNode(index, lower32 & Collation::ONLY_TERTIARY_MASK, | |
| UCOL_TERTIARY, errorCode); | |
| } | |
| } | |
| return index; | |
| } | |
| namespace { | |
| /** | |
| * Like Java Collections.binarySearch(List, key, Comparator). | |
| * | |
| * @return the index>=0 where the item was found, | |
| * or the index<0 for inserting the string at ~index in sorted order | |
| * (index into rootPrimaryIndexes) | |
| */ | |
| int32_t | |
| binarySearchForRootPrimaryNode(const int32_t *rootPrimaryIndexes, int32_t length, | |
| const int64_t *nodes, uint32_t p) { | |
| if(length == 0) { return ~0; } | |
| int32_t start = 0; | |
| int32_t limit = length; | |
| for (;;) { | |
| int32_t i = (start + limit) / 2; | |
| int64_t node = nodes[rootPrimaryIndexes[i]]; | |
| uint32_t nodePrimary = (uint32_t)(node >> 32); // weight32FromNode(node) | |
| if (p == nodePrimary) { | |
| return i; | |
| } else if (p < nodePrimary) { | |
| if (i == start) { | |
| return ~start; // insert s before i | |
| } | |
| limit = i; | |
| } else { | |
| if (i == start) { | |
| return ~(start + 1); // insert s after i | |
| } | |
| start = i; | |
| } | |
| } | |
| } | |
| } // namespace | |
| int32_t | |
| CollationBuilder::findOrInsertNodeForPrimary(uint32_t p, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return 0; } | |
| int32_t rootIndex = binarySearchForRootPrimaryNode( | |
| rootPrimaryIndexes.getBuffer(), rootPrimaryIndexes.size(), nodes.getBuffer(), p); | |
| if(rootIndex >= 0) { | |
| return rootPrimaryIndexes.elementAti(rootIndex); | |
| } else { | |
| // Start a new list of nodes with this primary. | |
| int32_t index = nodes.size(); | |
| nodes.addElement(nodeFromWeight32(p), errorCode); | |
| rootPrimaryIndexes.insertElementAt(index, ~rootIndex, errorCode); | |
| return index; | |
| } | |
| } | |
| int32_t | |
| CollationBuilder::findOrInsertWeakNode(int32_t index, uint32_t weight16, int32_t level, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return 0; } | |
| U_ASSERT(0 <= index && index < nodes.size()); | |
| U_ASSERT(UCOL_SECONDARY <= level && level <= UCOL_TERTIARY); | |
| if(weight16 == Collation::COMMON_WEIGHT16) { | |
| return findCommonNode(index, level); | |
| } | |
| // If this will be the first below-common weight for the parent node, | |
| // then we will also need to insert a common weight after it. | |
| int64_t node = nodes.elementAti(index); | |
| U_ASSERT(strengthFromNode(node) < level); // parent node is stronger | |
| if(weight16 != 0 && weight16 < Collation::COMMON_WEIGHT16) { | |
| int32_t hasThisLevelBefore = level == UCOL_SECONDARY ? HAS_BEFORE2 : HAS_BEFORE3; | |
| if((node & hasThisLevelBefore) == 0) { | |
| // The parent node has an implied level-common weight. | |
| int64_t commonNode = | |
| nodeFromWeight16(Collation::COMMON_WEIGHT16) | nodeFromStrength(level); | |
| if(level == UCOL_SECONDARY) { | |
| // Move the HAS_BEFORE3 flag from the parent node | |
| // to the new secondary common node. | |
| commonNode |= node & HAS_BEFORE3; | |
| node &= ~(int64_t)HAS_BEFORE3; | |
| } | |
| nodes.setElementAt(node | hasThisLevelBefore, index); | |
| // Insert below-common-weight node. | |
| int32_t nextIndex = nextIndexFromNode(node); | |
| node = nodeFromWeight16(weight16) | nodeFromStrength(level); | |
| index = insertNodeBetween(index, nextIndex, node, errorCode); | |
| // Insert common-weight node. | |
| insertNodeBetween(index, nextIndex, commonNode, errorCode); | |
| // Return index of below-common-weight node. | |
| return index; | |
| } | |
| } | |
| // Find the root CE's weight for this level. | |
| // Postpone insertion if not found: | |
| // Insert the new root node before the next stronger node, | |
| // or before the next root node with the same strength and a larger weight. | |
| int32_t nextIndex; | |
| while((nextIndex = nextIndexFromNode(node)) != 0) { | |
| node = nodes.elementAti(nextIndex); | |
| int32_t nextStrength = strengthFromNode(node); | |
| if(nextStrength <= level) { | |
| // Insert before a stronger node. | |
| if(nextStrength < level) { break; } | |
| // nextStrength == level | |
| if(!isTailoredNode(node)) { | |
| uint32_t nextWeight16 = weight16FromNode(node); | |
| if(nextWeight16 == weight16) { | |
| // Found the node for the root CE up to this level. | |
| return nextIndex; | |
| } | |
| // Insert before a node with a larger same-strength weight. | |
| if(nextWeight16 > weight16) { break; } | |
| } | |
| } | |
| // Skip the next node. | |
| index = nextIndex; | |
| } | |
| node = nodeFromWeight16(weight16) | nodeFromStrength(level); | |
| return insertNodeBetween(index, nextIndex, node, errorCode); | |
| } | |
| int32_t | |
| CollationBuilder::insertTailoredNodeAfter(int32_t index, int32_t strength, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return 0; } | |
| U_ASSERT(0 <= index && index < nodes.size()); | |
| if(strength >= UCOL_SECONDARY) { | |
| index = findCommonNode(index, UCOL_SECONDARY); | |
| if(strength >= UCOL_TERTIARY) { | |
| index = findCommonNode(index, UCOL_TERTIARY); | |
| } | |
| } | |
| // Postpone insertion: | |
| // Insert the new node before the next one with a strength at least as strong. | |
| int64_t node = nodes.elementAti(index); | |
| int32_t nextIndex; | |
| while((nextIndex = nextIndexFromNode(node)) != 0) { | |
| node = nodes.elementAti(nextIndex); | |
| if(strengthFromNode(node) <= strength) { break; } | |
| // Skip the next node which has a weaker (larger) strength than the new one. | |
| index = nextIndex; | |
| } | |
| node = IS_TAILORED | nodeFromStrength(strength); | |
| return insertNodeBetween(index, nextIndex, node, errorCode); | |
| } | |
| int32_t | |
| CollationBuilder::insertNodeBetween(int32_t index, int32_t nextIndex, int64_t node, | |
| UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return 0; } | |
| U_ASSERT(previousIndexFromNode(node) == 0); | |
| U_ASSERT(nextIndexFromNode(node) == 0); | |
| U_ASSERT(nextIndexFromNode(nodes.elementAti(index)) == nextIndex); | |
| // Append the new node and link it to the existing nodes. | |
| int32_t newIndex = nodes.size(); | |
| node |= nodeFromPreviousIndex(index) | nodeFromNextIndex(nextIndex); | |
| nodes.addElement(node, errorCode); | |
| if(U_FAILURE(errorCode)) { return 0; } | |
| // nodes[index].nextIndex = newIndex | |
| node = nodes.elementAti(index); | |
| nodes.setElementAt(changeNodeNextIndex(node, newIndex), index); | |
| // nodes[nextIndex].previousIndex = newIndex | |
| if(nextIndex != 0) { | |
| node = nodes.elementAti(nextIndex); | |
| nodes.setElementAt(changeNodePreviousIndex(node, newIndex), nextIndex); | |
| } | |
| return newIndex; | |
| } | |
| int32_t | |
| CollationBuilder::findCommonNode(int32_t index, int32_t strength) const { | |
| U_ASSERT(UCOL_SECONDARY <= strength && strength <= UCOL_TERTIARY); | |
| int64_t node = nodes.elementAti(index); | |
| if(strengthFromNode(node) >= strength) { | |
| // The current node is no stronger. | |
| return index; | |
| } | |
| if(strength == UCOL_SECONDARY ? !nodeHasBefore2(node) : !nodeHasBefore3(node)) { | |
| // The current node implies the strength-common weight. | |
| return index; | |
| } | |
| index = nextIndexFromNode(node); | |
| node = nodes.elementAti(index); | |
| U_ASSERT(!isTailoredNode(node) && strengthFromNode(node) == strength && | |
| weight16FromNode(node) < Collation::COMMON_WEIGHT16); | |
| // Skip to the explicit common node. | |
| do { | |
| index = nextIndexFromNode(node); | |
| node = nodes.elementAti(index); | |
| U_ASSERT(strengthFromNode(node) >= strength); | |
| } while(isTailoredNode(node) || strengthFromNode(node) > strength || | |
| weight16FromNode(node) < Collation::COMMON_WEIGHT16); | |
| U_ASSERT(weight16FromNode(node) == Collation::COMMON_WEIGHT16); | |
| return index; | |
| } | |
| void | |
| CollationBuilder::setCaseBits(const UnicodeString &nfdString, | |
| const char *&parserErrorReason, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| int32_t numTailoredPrimaries = 0; | |
| for(int32_t i = 0; i < cesLength; ++i) { | |
| if(ceStrength(ces[i]) == UCOL_PRIMARY) { ++numTailoredPrimaries; } | |
| } | |
| // We should not be able to get too many case bits because | |
| // cesLength<=31==MAX_EXPANSION_LENGTH. | |
| // 31 pairs of case bits fit into an int64_t without setting its sign bit. | |
| U_ASSERT(numTailoredPrimaries <= 31); | |
| int64_t cases = 0; | |
| if(numTailoredPrimaries > 0) { | |
| const UChar *s = nfdString.getBuffer(); | |
| UTF16CollationIterator baseCEs(baseData, FALSE, s, s, s + nfdString.length()); | |
| int32_t baseCEsLength = baseCEs.fetchCEs(errorCode) - 1; | |
| if(U_FAILURE(errorCode)) { | |
| parserErrorReason = "fetching root CEs for tailored string"; | |
| return; | |
| } | |
| U_ASSERT(baseCEsLength >= 0 && baseCEs.getCE(baseCEsLength) == Collation::NO_CE); | |
| uint32_t lastCase = 0; | |
| int32_t numBasePrimaries = 0; | |
| for(int32_t i = 0; i < baseCEsLength; ++i) { | |
| int64_t ce = baseCEs.getCE(i); | |
| if((ce >> 32) != 0) { | |
| ++numBasePrimaries; | |
| uint32_t c = ((uint32_t)ce >> 14) & 3; | |
| U_ASSERT(c == 0 || c == 2); // lowercase or uppercase, no mixed case in any base CE | |
| if(numBasePrimaries < numTailoredPrimaries) { | |
| cases |= (int64_t)c << ((numBasePrimaries - 1) * 2); | |
| } else if(numBasePrimaries == numTailoredPrimaries) { | |
| lastCase = c; | |
| } else if(c != lastCase) { | |
| // There are more base primary CEs than tailored primaries. | |
| // Set mixed case if the case bits of the remainder differ. | |
| lastCase = 1; | |
| // Nothing more can change. | |
| break; | |
| } | |
| } | |
| } | |
| if(numBasePrimaries >= numTailoredPrimaries) { | |
| cases |= (int64_t)lastCase << ((numTailoredPrimaries - 1) * 2); | |
| } | |
| } | |
| for(int32_t i = 0; i < cesLength; ++i) { | |
| int64_t ce = ces[i] & INT64_C(0xffffffffffff3fff); // clear old case bits | |
| int32_t strength = ceStrength(ce); | |
| if(strength == UCOL_PRIMARY) { | |
| ce |= (cases & 3) << 14; | |
| cases >>= 2; | |
| } else if(strength == UCOL_TERTIARY) { | |
| // Tertiary CEs must have uppercase bits. | |
| // See the LDML spec, and comments in class CollationCompare. | |
| ce |= 0x8000; | |
| } | |
| // Tertiary ignorable CEs must have 0 case bits. | |
| // We set 0 case bits for secondary CEs too | |
| // since currently only U+0345 is cased and maps to a secondary CE, | |
| // and it is lowercase. Other secondaries are uncased. | |
| // See [[:Cased:]&[:uca1=:]] where uca1 queries the root primary weight. | |
| ces[i] = ce; | |
| } | |
| } | |
| void | |
| CollationBuilder::suppressContractions(const UnicodeSet &set, const char *&parserErrorReason, | |
| UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| dataBuilder->suppressContractions(set, errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| parserErrorReason = "application of [suppressContractions [set]] failed"; | |
| } | |
| } | |
| void | |
| CollationBuilder::optimize(const UnicodeSet &set, const char *& /* parserErrorReason */, | |
| UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| optimizeSet.addAll(set); | |
| } | |
| uint32_t | |
| CollationBuilder::addWithClosure(const UnicodeString &nfdPrefix, const UnicodeString &nfdString, | |
| const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32, | |
| UErrorCode &errorCode) { | |
| // Map from the NFD input to the CEs. | |
| ce32 = addIfDifferent(nfdPrefix, nfdString, newCEs, newCEsLength, ce32, errorCode); | |
| ce32 = addOnlyClosure(nfdPrefix, nfdString, newCEs, newCEsLength, ce32, errorCode); | |
| addTailComposites(nfdPrefix, nfdString, errorCode); | |
| return ce32; | |
| } | |
| uint32_t | |
| CollationBuilder::addOnlyClosure(const UnicodeString &nfdPrefix, const UnicodeString &nfdString, | |
| const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32, | |
| UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return ce32; } | |
| // Map from canonically equivalent input to the CEs. (But not from the all-NFD input.) | |
| if(nfdPrefix.isEmpty()) { | |
| CanonicalIterator stringIter(nfdString, errorCode); | |
| if(U_FAILURE(errorCode)) { return ce32; } | |
| UnicodeString prefix; | |
| for(;;) { | |
| UnicodeString str = stringIter.next(); | |
| if(str.isBogus()) { break; } | |
| if(ignoreString(str, errorCode) || str == nfdString) { continue; } | |
| ce32 = addIfDifferent(prefix, str, newCEs, newCEsLength, ce32, errorCode); | |
| if(U_FAILURE(errorCode)) { return ce32; } | |
| } | |
| } else { | |
| CanonicalIterator prefixIter(nfdPrefix, errorCode); | |
| CanonicalIterator stringIter(nfdString, errorCode); | |
| if(U_FAILURE(errorCode)) { return ce32; } | |
| for(;;) { | |
| UnicodeString prefix = prefixIter.next(); | |
| if(prefix.isBogus()) { break; } | |
| if(ignorePrefix(prefix, errorCode)) { continue; } | |
| UBool samePrefix = prefix == nfdPrefix; | |
| for(;;) { | |
| UnicodeString str = stringIter.next(); | |
| if(str.isBogus()) { break; } | |
| if(ignoreString(str, errorCode) || (samePrefix && str == nfdString)) { continue; } | |
| ce32 = addIfDifferent(prefix, str, newCEs, newCEsLength, ce32, errorCode); | |
| if(U_FAILURE(errorCode)) { return ce32; } | |
| } | |
| stringIter.reset(); | |
| } | |
| } | |
| return ce32; | |
| } | |
| void | |
| CollationBuilder::addTailComposites(const UnicodeString &nfdPrefix, const UnicodeString &nfdString, | |
| UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| // Look for the last starter in the NFD string. | |
| UChar32 lastStarter; | |
| int32_t indexAfterLastStarter = nfdString.length(); | |
| for(;;) { | |
| if(indexAfterLastStarter == 0) { return; } // no starter at all | |
| lastStarter = nfdString.char32At(indexAfterLastStarter - 1); | |
| if(nfd.getCombiningClass(lastStarter) == 0) { break; } | |
| indexAfterLastStarter -= U16_LENGTH(lastStarter); | |
| } | |
| // No closure to Hangul syllables since we decompose them on the fly. | |
| if(Hangul::isJamoL(lastStarter)) { return; } | |
| // Are there any composites whose decomposition starts with the lastStarter? | |
| // Note: Normalizer2Impl does not currently return start sets for NFC_QC=Maybe characters. | |
| // We might find some more equivalent mappings here if it did. | |
| UnicodeSet composites; | |
| if(!nfcImpl.getCanonStartSet(lastStarter, composites)) { return; } | |
| UnicodeString decomp; | |
| UnicodeString newNFDString, newString; | |
| int64_t newCEs[Collation::MAX_EXPANSION_LENGTH]; | |
| UnicodeSetIterator iter(composites); | |
| while(iter.next()) { | |
| U_ASSERT(!iter.isString()); | |
| UChar32 composite = iter.getCodepoint(); | |
| nfd.getDecomposition(composite, decomp); | |
| if(!mergeCompositeIntoString(nfdString, indexAfterLastStarter, composite, decomp, | |
| newNFDString, newString, errorCode)) { | |
| continue; | |
| } | |
| int32_t newCEsLength = dataBuilder->getCEs(nfdPrefix, newNFDString, newCEs, 0); | |
| if(newCEsLength > Collation::MAX_EXPANSION_LENGTH) { | |
| // Ignore mappings that we cannot store. | |
| continue; | |
| } | |
| // Note: It is possible that the newCEs do not make use of the mapping | |
| // for which we are adding the tail composites, in which case we might be adding | |
| // unnecessary mappings. | |
| // For example, when we add tail composites for ae^ (^=combining circumflex), | |
| // UCA discontiguous-contraction matching does not find any matches | |
| // for ae_^ (_=any combining diacritic below) *unless* there is also | |
| // a contraction mapping for ae. | |
| // Thus, if there is no ae contraction, then the ae^ mapping is ignored | |
| // while fetching the newCEs for ae_^. | |
| // TODO: Try to detect this effectively. | |
| // (Alternatively, print a warning when prefix contractions are missing.) | |
| // We do not need an explicit mapping for the NFD strings. | |
| // It is fine if the NFD input collates like this via a sequence of mappings. | |
| // It also saves a little bit of space, and may reduce the set of characters with contractions. | |
| uint32_t ce32 = addIfDifferent(nfdPrefix, newString, | |
| newCEs, newCEsLength, Collation::UNASSIGNED_CE32, errorCode); | |
| if(ce32 != Collation::UNASSIGNED_CE32) { | |
| // was different, was added | |
| addOnlyClosure(nfdPrefix, newNFDString, newCEs, newCEsLength, ce32, errorCode); | |
| } | |
| } | |
| } | |
| UBool | |
| CollationBuilder::mergeCompositeIntoString(const UnicodeString &nfdString, | |
| int32_t indexAfterLastStarter, | |
| UChar32 composite, const UnicodeString &decomp, | |
| UnicodeString &newNFDString, UnicodeString &newString, | |
| UErrorCode &errorCode) const { | |
| if(U_FAILURE(errorCode)) { return FALSE; } | |
| U_ASSERT(nfdString.char32At(indexAfterLastStarter - 1) == decomp.char32At(0)); | |
| int32_t lastStarterLength = decomp.moveIndex32(0, 1); | |
| if(lastStarterLength == decomp.length()) { | |
| // Singleton decompositions should be found by addWithClosure() | |
| // and the CanonicalIterator, so we can ignore them here. | |
| return FALSE; | |
| } | |
| if(nfdString.compare(indexAfterLastStarter, 0x7fffffff, | |
| decomp, lastStarterLength, 0x7fffffff) == 0) { | |
| // same strings, nothing new to be found here | |
| return FALSE; | |
| } | |
| // Make new FCD strings that combine a composite, or its decomposition, | |
| // into the nfdString's last starter and the combining marks following it. | |
| // Make an NFD version, and a version with the composite. | |
| newNFDString.setTo(nfdString, 0, indexAfterLastStarter); | |
| newString.setTo(nfdString, 0, indexAfterLastStarter - lastStarterLength).append(composite); | |
| // The following is related to discontiguous contraction matching, | |
| // but builds only FCD strings (or else returns FALSE). | |
| int32_t sourceIndex = indexAfterLastStarter; | |
| int32_t decompIndex = lastStarterLength; | |
| // Small optimization: We keep the source character across loop iterations | |
| // because we do not always consume it, | |
| // and then need not fetch it again nor look up its combining class again. | |
| UChar32 sourceChar = U_SENTINEL; | |
| // The cc variables need to be declared before the loop so that at the end | |
| // they are set to the last combining classes seen. | |
| uint8_t sourceCC = 0; | |
| uint8_t decompCC = 0; | |
| for(;;) { | |
| if(sourceChar < 0) { | |
| if(sourceIndex >= nfdString.length()) { break; } | |
| sourceChar = nfdString.char32At(sourceIndex); | |
| sourceCC = nfd.getCombiningClass(sourceChar); | |
| U_ASSERT(sourceCC != 0); | |
| } | |
| // We consume a decomposition character in each iteration. | |
| if(decompIndex >= decomp.length()) { break; } | |
| UChar32 decompChar = decomp.char32At(decompIndex); | |
| decompCC = nfd.getCombiningClass(decompChar); | |
| // Compare the two characters and their combining classes. | |
| if(decompCC == 0) { | |
| // Unable to merge because the source contains a non-zero combining mark | |
| // but the composite's decomposition contains another starter. | |
| // The strings would not be equivalent. | |
| return FALSE; | |
| } else if(sourceCC < decompCC) { | |
| // Composite + sourceChar would not be FCD. | |
| return FALSE; | |
| } else if(decompCC < sourceCC) { | |
| newNFDString.append(decompChar); | |
| decompIndex += U16_LENGTH(decompChar); | |
| } else if(decompChar != sourceChar) { | |
| // Blocked because same combining class. | |
| return FALSE; | |
| } else { // match: decompChar == sourceChar | |
| newNFDString.append(decompChar); | |
| decompIndex += U16_LENGTH(decompChar); | |
| sourceIndex += U16_LENGTH(decompChar); | |
| sourceChar = U_SENTINEL; | |
| } | |
| } | |
| // We are at the end of at least one of the two inputs. | |
| if(sourceChar >= 0) { // more characters from nfdString but not from decomp | |
| if(sourceCC < decompCC) { | |
| // Appending the next source character to the composite would not be FCD. | |
| return FALSE; | |
| } | |
| newNFDString.append(nfdString, sourceIndex, 0x7fffffff); | |
| newString.append(nfdString, sourceIndex, 0x7fffffff); | |
| } else if(decompIndex < decomp.length()) { // more characters from decomp, not from nfdString | |
| newNFDString.append(decomp, decompIndex, 0x7fffffff); | |
| } | |
| U_ASSERT(nfd.isNormalized(newNFDString, errorCode)); | |
| U_ASSERT(fcd.isNormalized(newString, errorCode)); | |
| U_ASSERT(nfd.normalize(newString, errorCode) == newNFDString); // canonically equivalent | |
| return TRUE; | |
| } | |
| UBool | |
| CollationBuilder::ignorePrefix(const UnicodeString &s, UErrorCode &errorCode) const { | |
| // Do not map non-FCD prefixes. | |
| return !isFCD(s, errorCode); | |
| } | |
| UBool | |
| CollationBuilder::ignoreString(const UnicodeString &s, UErrorCode &errorCode) const { | |
| // Do not map non-FCD strings. | |
| // Do not map strings that start with Hangul syllables: We decompose those on the fly. | |
| return !isFCD(s, errorCode) || Hangul::isHangul(s.charAt(0)); | |
| } | |
| UBool | |
| CollationBuilder::isFCD(const UnicodeString &s, UErrorCode &errorCode) const { | |
| return U_SUCCESS(errorCode) && fcd.isNormalized(s, errorCode); | |
| } | |
| void | |
| CollationBuilder::closeOverComposites(UErrorCode &errorCode) { | |
| UnicodeSet composites(UNICODE_STRING_SIMPLE("[:NFD_QC=N:]"), errorCode); // Java: static final | |
| if(U_FAILURE(errorCode)) { return; } | |
| // Hangul is decomposed on the fly during collation. | |
| composites.remove(Hangul::HANGUL_BASE, Hangul::HANGUL_END); | |
| UnicodeString prefix; // empty | |
| UnicodeString nfdString; | |
| UnicodeSetIterator iter(composites); | |
| while(iter.next()) { | |
| U_ASSERT(!iter.isString()); | |
| nfd.getDecomposition(iter.getCodepoint(), nfdString); | |
| cesLength = dataBuilder->getCEs(nfdString, ces, 0); | |
| if(cesLength > Collation::MAX_EXPANSION_LENGTH) { | |
| // Too many CEs from the decomposition (unusual), ignore this composite. | |
| // We could add a capacity parameter to getCEs() and reallocate if necessary. | |
| // However, this can only really happen in contrived cases. | |
| continue; | |
| } | |
| const UnicodeString &composite(iter.getString()); | |
| addIfDifferent(prefix, composite, ces, cesLength, Collation::UNASSIGNED_CE32, errorCode); | |
| } | |
| } | |
| uint32_t | |
| CollationBuilder::addIfDifferent(const UnicodeString &prefix, const UnicodeString &str, | |
| const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32, | |
| UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return ce32; } | |
| int64_t oldCEs[Collation::MAX_EXPANSION_LENGTH]; | |
| int32_t oldCEsLength = dataBuilder->getCEs(prefix, str, oldCEs, 0); | |
| if(!sameCEs(newCEs, newCEsLength, oldCEs, oldCEsLength)) { | |
| if(ce32 == Collation::UNASSIGNED_CE32) { | |
| ce32 = dataBuilder->encodeCEs(newCEs, newCEsLength, errorCode); | |
| } | |
| dataBuilder->addCE32(prefix, str, ce32, errorCode); | |
| } | |
| return ce32; | |
| } | |
| UBool | |
| CollationBuilder::sameCEs(const int64_t ces1[], int32_t ces1Length, | |
| const int64_t ces2[], int32_t ces2Length) { | |
| if(ces1Length != ces2Length) { | |
| return FALSE; | |
| } | |
| U_ASSERT(ces1Length <= Collation::MAX_EXPANSION_LENGTH); | |
| for(int32_t i = 0; i < ces1Length; ++i) { | |
| if(ces1[i] != ces2[i]) { return FALSE; } | |
| } | |
| return TRUE; | |
| } | |
| #ifdef DEBUG_COLLATION_BUILDER | |
| uint32_t | |
| alignWeightRight(uint32_t w) { | |
| if(w != 0) { | |
| while((w & 0xff) == 0) { w >>= 8; } | |
| } | |
| return w; | |
| } | |
| #endif | |
| void | |
| CollationBuilder::makeTailoredCEs(UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| CollationWeights primaries, secondaries, tertiaries; | |
| int64_t *nodesArray = nodes.getBuffer(); | |
| #ifdef DEBUG_COLLATION_BUILDER | |
| puts("\nCollationBuilder::makeTailoredCEs()"); | |
| #endif | |
| for(int32_t rpi = 0; rpi < rootPrimaryIndexes.size(); ++rpi) { | |
| int32_t i = rootPrimaryIndexes.elementAti(rpi); | |
| int64_t node = nodesArray[i]; | |
| uint32_t p = weight32FromNode(node); | |
| uint32_t s = p == 0 ? 0 : Collation::COMMON_WEIGHT16; | |
| uint32_t t = s; | |
| uint32_t q = 0; | |
| UBool pIsTailored = FALSE; | |
| UBool sIsTailored = FALSE; | |
| UBool tIsTailored = FALSE; | |
| #ifdef DEBUG_COLLATION_BUILDER | |
| printf("\nprimary %lx\n", (long)alignWeightRight(p)); | |
| #endif | |
| int32_t pIndex = p == 0 ? 0 : rootElements.findPrimary(p); | |
| int32_t nextIndex = nextIndexFromNode(node); | |
| while(nextIndex != 0) { | |
| i = nextIndex; | |
| node = nodesArray[i]; | |
| nextIndex = nextIndexFromNode(node); | |
| int32_t strength = strengthFromNode(node); | |
| if(strength == UCOL_QUATERNARY) { | |
| U_ASSERT(isTailoredNode(node)); | |
| #ifdef DEBUG_COLLATION_BUILDER | |
| printf(" quat+ "); | |
| #endif | |
| if(q == 3) { | |
| errorCode = U_BUFFER_OVERFLOW_ERROR; | |
| errorReason = "quaternary tailoring gap too small"; | |
| return; | |
| } | |
| ++q; | |
| } else { | |
| if(strength == UCOL_TERTIARY) { | |
| if(isTailoredNode(node)) { | |
| #ifdef DEBUG_COLLATION_BUILDER | |
| printf(" ter+ "); | |
| #endif | |
| if(!tIsTailored) { | |
| // First tailored tertiary node for [p, s]. | |
| int32_t tCount = countTailoredNodes(nodesArray, nextIndex, | |
| UCOL_TERTIARY) + 1; | |
| uint32_t tLimit; | |
| if(t == 0) { | |
| // Gap at the beginning of the tertiary CE range. | |
| t = rootElements.getTertiaryBoundary() - 0x100; | |
| tLimit = rootElements.getFirstTertiaryCE() & Collation::ONLY_TERTIARY_MASK; | |
| } else if(!pIsTailored && !sIsTailored) { | |
| // p and s are root weights. | |
| tLimit = rootElements.getTertiaryAfter(pIndex, s, t); | |
| } else if(t == Collation::BEFORE_WEIGHT16) { | |
| tLimit = Collation::COMMON_WEIGHT16; | |
| } else { | |
| // [p, s] is tailored. | |
| U_ASSERT(t == Collation::COMMON_WEIGHT16); | |
| tLimit = rootElements.getTertiaryBoundary(); | |
| } | |
| U_ASSERT(tLimit == 0x4000 || (tLimit & ~Collation::ONLY_TERTIARY_MASK) == 0); | |
| tertiaries.initForTertiary(); | |
| if(!tertiaries.allocWeights(t, tLimit, tCount)) { | |
| errorCode = U_BUFFER_OVERFLOW_ERROR; | |
| errorReason = "tertiary tailoring gap too small"; | |
| return; | |
| } | |
| tIsTailored = TRUE; | |
| } | |
| t = tertiaries.nextWeight(); | |
| U_ASSERT(t != 0xffffffff); | |
| } else { | |
| t = weight16FromNode(node); | |
| tIsTailored = FALSE; | |
| #ifdef DEBUG_COLLATION_BUILDER | |
| printf(" ter %lx\n", (long)alignWeightRight(t)); | |
| #endif | |
| } | |
| } else { | |
| if(strength == UCOL_SECONDARY) { | |
| if(isTailoredNode(node)) { | |
| #ifdef DEBUG_COLLATION_BUILDER | |
| printf(" sec+ "); | |
| #endif | |
| if(!sIsTailored) { | |
| // First tailored secondary node for p. | |
| int32_t sCount = countTailoredNodes(nodesArray, nextIndex, | |
| UCOL_SECONDARY) + 1; | |
| uint32_t sLimit; | |
| if(s == 0) { | |
| // Gap at the beginning of the secondary CE range. | |
| s = rootElements.getSecondaryBoundary() - 0x100; | |
| sLimit = rootElements.getFirstSecondaryCE() >> 16; | |
| } else if(!pIsTailored) { | |
| // p is a root primary. | |
| sLimit = rootElements.getSecondaryAfter(pIndex, s); | |
| } else if(s == Collation::BEFORE_WEIGHT16) { | |
| sLimit = Collation::COMMON_WEIGHT16; | |
| } else { | |
| // p is a tailored primary. | |
| U_ASSERT(s == Collation::COMMON_WEIGHT16); | |
| sLimit = rootElements.getSecondaryBoundary(); | |
| } | |
| if(s == Collation::COMMON_WEIGHT16) { | |
| // Do not tailor into the getSortKey() range of | |
| // compressed common secondaries. | |
| s = rootElements.getLastCommonSecondary(); | |
| } | |
| secondaries.initForSecondary(); | |
| if(!secondaries.allocWeights(s, sLimit, sCount)) { | |
| errorCode = U_BUFFER_OVERFLOW_ERROR; | |
| errorReason = "secondary tailoring gap too small"; | |
| #ifdef DEBUG_COLLATION_BUILDER | |
| printf("!secondaries.allocWeights(%lx, %lx, sCount=%ld)\n", | |
| (long)alignWeightRight(s), (long)alignWeightRight(sLimit), | |
| (long)alignWeightRight(sCount)); | |
| #endif | |
| return; | |
| } | |
| sIsTailored = TRUE; | |
| } | |
| s = secondaries.nextWeight(); | |
| U_ASSERT(s != 0xffffffff); | |
| } else { | |
| s = weight16FromNode(node); | |
| sIsTailored = FALSE; | |
| #ifdef DEBUG_COLLATION_BUILDER | |
| printf(" sec %lx\n", (long)alignWeightRight(s)); | |
| #endif | |
| } | |
| } else /* UCOL_PRIMARY */ { | |
| U_ASSERT(isTailoredNode(node)); | |
| #ifdef DEBUG_COLLATION_BUILDER | |
| printf("pri+ "); | |
| #endif | |
| if(!pIsTailored) { | |
| // First tailored primary node in this list. | |
| int32_t pCount = countTailoredNodes(nodesArray, nextIndex, | |
| UCOL_PRIMARY) + 1; | |
| UBool isCompressible = baseData->isCompressiblePrimary(p); | |
| uint32_t pLimit = | |
| rootElements.getPrimaryAfter(p, pIndex, isCompressible); | |
| primaries.initForPrimary(isCompressible); | |
| if(!primaries.allocWeights(p, pLimit, pCount)) { | |
| errorCode = U_BUFFER_OVERFLOW_ERROR; // TODO: introduce a more specific UErrorCode? | |
| errorReason = "primary tailoring gap too small"; | |
| return; | |
| } | |
| pIsTailored = TRUE; | |
| } | |
| p = primaries.nextWeight(); | |
| U_ASSERT(p != 0xffffffff); | |
| s = Collation::COMMON_WEIGHT16; | |
| sIsTailored = FALSE; | |
| } | |
| t = s == 0 ? 0 : Collation::COMMON_WEIGHT16; | |
| tIsTailored = FALSE; | |
| } | |
| q = 0; | |
| } | |
| if(isTailoredNode(node)) { | |
| nodesArray[i] = Collation::makeCE(p, s, t, q); | |
| #ifdef DEBUG_COLLATION_BUILDER | |
| printf("%016llx\n", (long long)nodesArray[i]); | |
| #endif | |
| } | |
| } | |
| } | |
| } | |
| int32_t | |
| CollationBuilder::countTailoredNodes(const int64_t *nodesArray, int32_t i, int32_t strength) { | |
| int32_t count = 0; | |
| for(;;) { | |
| if(i == 0) { break; } | |
| int64_t node = nodesArray[i]; | |
| if(strengthFromNode(node) < strength) { break; } | |
| if(strengthFromNode(node) == strength) { | |
| if(isTailoredNode(node)) { | |
| ++count; | |
| } else { | |
| break; | |
| } | |
| } | |
| i = nextIndexFromNode(node); | |
| } | |
| return count; | |
| } | |
| class CEFinalizer : public CollationDataBuilder::CEModifier { | |
| public: | |
| CEFinalizer(const int64_t *ces) : finalCEs(ces) {} | |
| virtual ~CEFinalizer(); | |
| virtual int64_t modifyCE32(uint32_t ce32) const { | |
| U_ASSERT(!Collation::isSpecialCE32(ce32)); | |
| if(CollationBuilder::isTempCE32(ce32)) { | |
| // retain case bits | |
| return finalCEs[CollationBuilder::indexFromTempCE32(ce32)] | ((ce32 & 0xc0) << 8); | |
| } else { | |
| return Collation::NO_CE; | |
| } | |
| } | |
| virtual int64_t modifyCE(int64_t ce) const { | |
| if(CollationBuilder::isTempCE(ce)) { | |
| // retain case bits | |
| return finalCEs[CollationBuilder::indexFromTempCE(ce)] | (ce & 0xc000); | |
| } else { | |
| return Collation::NO_CE; | |
| } | |
| } | |
| private: | |
| const int64_t *finalCEs; | |
| }; | |
| CEFinalizer::~CEFinalizer() {} | |
| void | |
| CollationBuilder::finalizeCEs(UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| LocalPointer<CollationDataBuilder> newBuilder(new CollationDataBuilder(errorCode), errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| return; | |
| } | |
| newBuilder->initForTailoring(baseData, errorCode); | |
| CEFinalizer finalizer(nodes.getBuffer()); | |
| newBuilder->copyFrom(*dataBuilder, finalizer, errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| delete dataBuilder; | |
| dataBuilder = newBuilder.orphan(); | |
| } | |
| int32_t | |
| CollationBuilder::ceStrength(int64_t ce) { | |
| return | |
| isTempCE(ce) ? strengthFromTempCE(ce) : | |
| (ce & INT64_C(0xff00000000000000)) != 0 ? UCOL_PRIMARY : | |
| ((uint32_t)ce & 0xff000000) != 0 ? UCOL_SECONDARY : | |
| ce != 0 ? UCOL_TERTIARY : | |
| UCOL_IDENTICAL; | |
| } | |
| U_NAMESPACE_END | |
| U_NAMESPACE_USE | |
| U_CAPI UCollator * U_EXPORT2 | |
| ucol_openRules(const UChar *rules, int32_t rulesLength, | |
| UColAttributeValue normalizationMode, UCollationStrength strength, | |
| UParseError *parseError, UErrorCode *pErrorCode) { | |
| if(U_FAILURE(*pErrorCode)) { return NULL; } | |
| if(rules == NULL && rulesLength != 0) { | |
| *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; | |
| return NULL; | |
| } | |
| RuleBasedCollator *coll = new RuleBasedCollator(); | |
| if(coll == NULL) { | |
| *pErrorCode = U_MEMORY_ALLOCATION_ERROR; | |
| return NULL; | |
| } | |
| UnicodeString r((UBool)(rulesLength < 0), rules, rulesLength); | |
| coll->internalBuildTailoring(r, strength, normalizationMode, parseError, NULL, *pErrorCode); | |
| if(U_FAILURE(*pErrorCode)) { | |
| delete coll; | |
| return NULL; | |
| } | |
| return coll->toUCollator(); | |
| } | |
| static const int32_t internalBufferSize = 512; | |
| // The @internal ucol_getUnsafeSet() was moved here from ucol_sit.cpp | |
| // because it calls UnicodeSet "builder" code that depends on all Unicode properties, | |
| // and the rest of the collation "runtime" code only depends on normalization. | |
| // This function is not related to the collation builder, | |
| // but it did not seem worth moving it into its own .cpp file, | |
| // nor rewriting it to use lower-level UnicodeSet and Normalizer2Impl methods. | |
| U_CAPI int32_t U_EXPORT2 | |
| ucol_getUnsafeSet( const UCollator *coll, | |
| USet *unsafe, | |
| UErrorCode *status) | |
| { | |
| UChar buffer[internalBufferSize]; | |
| int32_t len = 0; | |
| uset_clear(unsafe); | |
| // cccpattern = "[[:^tccc=0:][:^lccc=0:]]", unfortunately variant | |
| static const UChar cccpattern[25] = { 0x5b, 0x5b, 0x3a, 0x5e, 0x74, 0x63, 0x63, 0x63, 0x3d, 0x30, 0x3a, 0x5d, | |
| 0x5b, 0x3a, 0x5e, 0x6c, 0x63, 0x63, 0x63, 0x3d, 0x30, 0x3a, 0x5d, 0x5d, 0x00 }; | |
| // add chars that fail the fcd check | |
| uset_applyPattern(unsafe, cccpattern, 24, USET_IGNORE_SPACE, status); | |
| // add lead/trail surrogates | |
| // (trail surrogates should need to be unsafe only if the caller tests for UTF-16 code *units*, | |
| // not when testing code *points*) | |
| uset_addRange(unsafe, 0xd800, 0xdfff); | |
| USet *contractions = uset_open(0,0); | |
| int32_t i = 0, j = 0; | |
| ucol_getContractionsAndExpansions(coll, contractions, NULL, FALSE, status); | |
| int32_t contsSize = uset_size(contractions); | |
| UChar32 c = 0; | |
| // Contraction set consists only of strings | |
| // to get unsafe code points, we need to | |
| // break the strings apart and add them to the unsafe set | |
| for(i = 0; i < contsSize; i++) { | |
| len = uset_getItem(contractions, i, NULL, NULL, buffer, internalBufferSize, status); | |
| if(len > 0) { | |
| j = 0; | |
| while(j < len) { | |
| U16_NEXT(buffer, j, len, c); | |
| if(j < len) { | |
| uset_add(unsafe, c); | |
| } | |
| } | |
| } | |
| } | |
| uset_close(contractions); | |
| return uset_size(unsafe); | |
| } | |
| #endif // !UCONFIG_NO_COLLATION |