| /* | |
| ******************************************************************************* | |
| * Copyright (C) 2013-2015, International Business Machines | |
| * Corporation and others. All Rights Reserved. | |
| ******************************************************************************* | |
| * collationruleparser.cpp | |
| * | |
| * (replaced the former ucol_tok.cpp) | |
| * | |
| * created on: 2013apr10 | |
| * created by: Markus W. Scherer | |
| */ | |
| #include "unicode/utypes.h" | |
| #if !UCONFIG_NO_COLLATION | |
| #include "starboard/client_porting/poem/assert_poem.h" | |
| #include "starboard/client_porting/poem/string_poem.h" | |
| #include "unicode/normalizer2.h" | |
| #include "unicode/parseerr.h" | |
| #include "unicode/uchar.h" | |
| #include "unicode/ucol.h" | |
| #include "unicode/uloc.h" | |
| #include "unicode/unistr.h" | |
| #include "unicode/utf16.h" | |
| #include "charstr.h" | |
| #include "cmemory.h" | |
| #include "collation.h" | |
| #include "collationdata.h" | |
| #include "collationruleparser.h" | |
| #include "collationsettings.h" | |
| #include "collationtailoring.h" | |
| #include "cstring.h" | |
| #include "patternprops.h" | |
| #include "uassert.h" | |
| #include "uvectr32.h" | |
| U_NAMESPACE_BEGIN | |
| namespace { | |
| static const UChar BEFORE[] = { 0x5b, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, 0 }; // "[before" | |
| const int32_t BEFORE_LENGTH = 7; | |
| } // namespace | |
| CollationRuleParser::Sink::~Sink() {} | |
| void | |
| CollationRuleParser::Sink::suppressContractions(const UnicodeSet &, const char *&, UErrorCode &) {} | |
| void | |
| CollationRuleParser::Sink::optimize(const UnicodeSet &, const char *&, UErrorCode &) {} | |
| CollationRuleParser::Importer::~Importer() {} | |
| CollationRuleParser::CollationRuleParser(const CollationData *base, UErrorCode &errorCode) | |
| : nfd(*Normalizer2::getNFDInstance(errorCode)), | |
| nfc(*Normalizer2::getNFCInstance(errorCode)), | |
| rules(NULL), baseData(base), settings(NULL), | |
| parseError(NULL), errorReason(NULL), | |
| sink(NULL), importer(NULL), | |
| ruleIndex(0) { | |
| } | |
| CollationRuleParser::~CollationRuleParser() { | |
| } | |
| void | |
| CollationRuleParser::parse(const UnicodeString &ruleString, | |
| CollationSettings &outSettings, | |
| UParseError *outParseError, | |
| UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| settings = &outSettings; | |
| parseError = outParseError; | |
| if(parseError != NULL) { | |
| parseError->line = 0; | |
| parseError->offset = -1; | |
| parseError->preContext[0] = 0; | |
| parseError->postContext[0] = 0; | |
| } | |
| errorReason = NULL; | |
| parse(ruleString, errorCode); | |
| } | |
| void | |
| CollationRuleParser::parse(const UnicodeString &ruleString, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| rules = &ruleString; | |
| ruleIndex = 0; | |
| while(ruleIndex < rules->length()) { | |
| UChar c = rules->charAt(ruleIndex); | |
| if(PatternProps::isWhiteSpace(c)) { | |
| ++ruleIndex; | |
| continue; | |
| } | |
| switch(c) { | |
| case 0x26: // '&' | |
| parseRuleChain(errorCode); | |
| break; | |
| case 0x5b: // '[' | |
| parseSetting(errorCode); | |
| break; | |
| case 0x23: // '#' starts a comment, until the end of the line | |
| ruleIndex = skipComment(ruleIndex + 1); | |
| break; | |
| case 0x40: // '@' is equivalent to [backwards 2] | |
| settings->setFlag(CollationSettings::BACKWARD_SECONDARY, | |
| UCOL_ON, 0, errorCode); | |
| ++ruleIndex; | |
| break; | |
| case 0x21: // '!' used to turn on Thai/Lao character reversal | |
| // Accept but ignore. The root collator has contractions | |
| // that are equivalent to the character reversal, where appropriate. | |
| ++ruleIndex; | |
| break; | |
| default: | |
| setParseError("expected a reset or setting or comment", errorCode); | |
| break; | |
| } | |
| if(U_FAILURE(errorCode)) { return; } | |
| } | |
| } | |
| void | |
| CollationRuleParser::parseRuleChain(UErrorCode &errorCode) { | |
| int32_t resetStrength = parseResetAndPosition(errorCode); | |
| UBool isFirstRelation = TRUE; | |
| for(;;) { | |
| int32_t result = parseRelationOperator(errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| if(result < 0) { | |
| if(ruleIndex < rules->length() && rules->charAt(ruleIndex) == 0x23) { | |
| // '#' starts a comment, until the end of the line | |
| ruleIndex = skipComment(ruleIndex + 1); | |
| continue; | |
| } | |
| if(isFirstRelation) { | |
| setParseError("reset not followed by a relation", errorCode); | |
| } | |
| return; | |
| } | |
| int32_t strength = result & STRENGTH_MASK; | |
| if(resetStrength < UCOL_IDENTICAL) { | |
| // reset-before rule chain | |
| if(isFirstRelation) { | |
| if(strength != resetStrength) { | |
| setParseError("reset-before strength differs from its first relation", errorCode); | |
| return; | |
| } | |
| } else { | |
| if(strength < resetStrength) { | |
| setParseError("reset-before strength followed by a stronger relation", errorCode); | |
| return; | |
| } | |
| } | |
| } | |
| int32_t i = ruleIndex + (result >> OFFSET_SHIFT); // skip over the relation operator | |
| if((result & STARRED_FLAG) == 0) { | |
| parseRelationStrings(strength, i, errorCode); | |
| } else { | |
| parseStarredCharacters(strength, i, errorCode); | |
| } | |
| if(U_FAILURE(errorCode)) { return; } | |
| isFirstRelation = FALSE; | |
| } | |
| } | |
| int32_t | |
| CollationRuleParser::parseResetAndPosition(UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return UCOL_DEFAULT; } | |
| int32_t i = skipWhiteSpace(ruleIndex + 1); | |
| int32_t j; | |
| UChar c; | |
| int32_t resetStrength; | |
| if(rules->compare(i, BEFORE_LENGTH, BEFORE, 0, BEFORE_LENGTH) == 0 && | |
| (j = i + BEFORE_LENGTH) < rules->length() && | |
| PatternProps::isWhiteSpace(rules->charAt(j)) && | |
| ((j = skipWhiteSpace(j + 1)) + 1) < rules->length() && | |
| 0x31 <= (c = rules->charAt(j)) && c <= 0x33 && | |
| rules->charAt(j + 1) == 0x5d) { | |
| // &[before n] with n=1 or 2 or 3 | |
| resetStrength = UCOL_PRIMARY + (c - 0x31); | |
| i = skipWhiteSpace(j + 2); | |
| } else { | |
| resetStrength = UCOL_IDENTICAL; | |
| } | |
| if(i >= rules->length()) { | |
| setParseError("reset without position", errorCode); | |
| return UCOL_DEFAULT; | |
| } | |
| UnicodeString str; | |
| if(rules->charAt(i) == 0x5b) { // '[' | |
| i = parseSpecialPosition(i, str, errorCode); | |
| } else { | |
| i = parseTailoringString(i, str, errorCode); | |
| } | |
| sink->addReset(resetStrength, str, errorReason, errorCode); | |
| if(U_FAILURE(errorCode)) { setErrorContext(); } | |
| ruleIndex = i; | |
| return resetStrength; | |
| } | |
| int32_t | |
| CollationRuleParser::parseRelationOperator(UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return UCOL_DEFAULT; } | |
| ruleIndex = skipWhiteSpace(ruleIndex); | |
| if(ruleIndex >= rules->length()) { return UCOL_DEFAULT; } | |
| int32_t strength; | |
| int32_t i = ruleIndex; | |
| UChar c = rules->charAt(i++); | |
| switch(c) { | |
| case 0x3c: // '<' | |
| if(i < rules->length() && rules->charAt(i) == 0x3c) { // << | |
| ++i; | |
| if(i < rules->length() && rules->charAt(i) == 0x3c) { // <<< | |
| ++i; | |
| if(i < rules->length() && rules->charAt(i) == 0x3c) { // <<<< | |
| ++i; | |
| strength = UCOL_QUATERNARY; | |
| } else { | |
| strength = UCOL_TERTIARY; | |
| } | |
| } else { | |
| strength = UCOL_SECONDARY; | |
| } | |
| } else { | |
| strength = UCOL_PRIMARY; | |
| } | |
| if(i < rules->length() && rules->charAt(i) == 0x2a) { // '*' | |
| ++i; | |
| strength |= STARRED_FLAG; | |
| } | |
| break; | |
| case 0x3b: // ';' same as << | |
| strength = UCOL_SECONDARY; | |
| break; | |
| case 0x2c: // ',' same as <<< | |
| strength = UCOL_TERTIARY; | |
| break; | |
| case 0x3d: // '=' | |
| strength = UCOL_IDENTICAL; | |
| if(i < rules->length() && rules->charAt(i) == 0x2a) { // '*' | |
| ++i; | |
| strength |= STARRED_FLAG; | |
| } | |
| break; | |
| default: | |
| return UCOL_DEFAULT; | |
| } | |
| return ((i - ruleIndex) << OFFSET_SHIFT) | strength; | |
| } | |
| void | |
| CollationRuleParser::parseRelationStrings(int32_t strength, int32_t i, UErrorCode &errorCode) { | |
| // Parse | |
| // prefix | str / extension | |
| // where prefix and extension are optional. | |
| UnicodeString prefix, str, extension; | |
| i = parseTailoringString(i, str, errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| UChar next = (i < rules->length()) ? rules->charAt(i) : 0; | |
| if(next == 0x7c) { // '|' separates the context prefix from the string. | |
| prefix = str; | |
| i = parseTailoringString(i + 1, str, errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| next = (i < rules->length()) ? rules->charAt(i) : 0; | |
| } | |
| if(next == 0x2f) { // '/' separates the string from the extension. | |
| i = parseTailoringString(i + 1, extension, errorCode); | |
| } | |
| if(!prefix.isEmpty()) { | |
| UChar32 prefix0 = prefix.char32At(0); | |
| UChar32 c = str.char32At(0); | |
| if(!nfc.hasBoundaryBefore(prefix0) || !nfc.hasBoundaryBefore(c)) { | |
| setParseError("in 'prefix|str', prefix and str must each start with an NFC boundary", | |
| errorCode); | |
| return; | |
| } | |
| } | |
| sink->addRelation(strength, prefix, str, extension, errorReason, errorCode); | |
| if(U_FAILURE(errorCode)) { setErrorContext(); } | |
| ruleIndex = i; | |
| } | |
| void | |
| CollationRuleParser::parseStarredCharacters(int32_t strength, int32_t i, UErrorCode &errorCode) { | |
| UnicodeString empty, raw; | |
| i = parseString(skipWhiteSpace(i), raw, errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| if(raw.isEmpty()) { | |
| setParseError("missing starred-relation string", errorCode); | |
| return; | |
| } | |
| UChar32 prev = -1; | |
| int32_t j = 0; | |
| for(;;) { | |
| while(j < raw.length()) { | |
| UChar32 c = raw.char32At(j); | |
| if(!nfd.isInert(c)) { | |
| setParseError("starred-relation string is not all NFD-inert", errorCode); | |
| return; | |
| } | |
| sink->addRelation(strength, empty, UnicodeString(c), empty, errorReason, errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| setErrorContext(); | |
| return; | |
| } | |
| j += U16_LENGTH(c); | |
| prev = c; | |
| } | |
| if(i >= rules->length() || rules->charAt(i) != 0x2d) { // '-' | |
| break; | |
| } | |
| if(prev < 0) { | |
| setParseError("range without start in starred-relation string", errorCode); | |
| return; | |
| } | |
| i = parseString(i + 1, raw, errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| if(raw.isEmpty()) { | |
| setParseError("range without end in starred-relation string", errorCode); | |
| return; | |
| } | |
| UChar32 c = raw.char32At(0); | |
| if(c < prev) { | |
| setParseError("range start greater than end in starred-relation string", errorCode); | |
| return; | |
| } | |
| // range prev-c | |
| UnicodeString s; | |
| while(++prev <= c) { | |
| if(!nfd.isInert(prev)) { | |
| setParseError("starred-relation string range is not all NFD-inert", errorCode); | |
| return; | |
| } | |
| if(U_IS_SURROGATE(prev)) { | |
| setParseError("starred-relation string range contains a surrogate", errorCode); | |
| return; | |
| } | |
| if(0xfffd <= prev && prev <= 0xffff) { | |
| setParseError("starred-relation string range contains U+FFFD, U+FFFE or U+FFFF", errorCode); | |
| return; | |
| } | |
| s.setTo(prev); | |
| sink->addRelation(strength, empty, s, empty, errorReason, errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| setErrorContext(); | |
| return; | |
| } | |
| } | |
| prev = -1; | |
| j = U16_LENGTH(c); | |
| } | |
| ruleIndex = skipWhiteSpace(i); | |
| } | |
| int32_t | |
| CollationRuleParser::parseTailoringString(int32_t i, UnicodeString &raw, UErrorCode &errorCode) { | |
| i = parseString(skipWhiteSpace(i), raw, errorCode); | |
| if(U_SUCCESS(errorCode) && raw.isEmpty()) { | |
| setParseError("missing relation string", errorCode); | |
| } | |
| return skipWhiteSpace(i); | |
| } | |
| int32_t | |
| CollationRuleParser::parseString(int32_t i, UnicodeString &raw, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return i; } | |
| raw.remove(); | |
| while(i < rules->length()) { | |
| UChar32 c = rules->charAt(i++); | |
| if(isSyntaxChar(c)) { | |
| if(c == 0x27) { // apostrophe | |
| if(i < rules->length() && rules->charAt(i) == 0x27) { | |
| // Double apostrophe, encodes a single one. | |
| raw.append((UChar)0x27); | |
| ++i; | |
| continue; | |
| } | |
| // Quote literal text until the next single apostrophe. | |
| for(;;) { | |
| if(i == rules->length()) { | |
| setParseError("quoted literal text missing terminating apostrophe", errorCode); | |
| return i; | |
| } | |
| c = rules->charAt(i++); | |
| if(c == 0x27) { | |
| if(i < rules->length() && rules->charAt(i) == 0x27) { | |
| // Double apostrophe inside quoted literal text, | |
| // still encodes a single apostrophe. | |
| ++i; | |
| } else { | |
| break; | |
| } | |
| } | |
| raw.append((UChar)c); | |
| } | |
| } else if(c == 0x5c) { // backslash | |
| if(i == rules->length()) { | |
| setParseError("backslash escape at the end of the rule string", errorCode); | |
| return i; | |
| } | |
| c = rules->char32At(i); | |
| raw.append(c); | |
| i += U16_LENGTH(c); | |
| } else { | |
| // Any other syntax character terminates a string. | |
| --i; | |
| break; | |
| } | |
| } else if(PatternProps::isWhiteSpace(c)) { | |
| // Unquoted white space terminates a string. | |
| --i; | |
| break; | |
| } else { | |
| raw.append((UChar)c); | |
| } | |
| } | |
| for(int32_t j = 0; j < raw.length();) { | |
| UChar32 c = raw.char32At(j); | |
| if(U_IS_SURROGATE(c)) { | |
| setParseError("string contains an unpaired surrogate", errorCode); | |
| return i; | |
| } | |
| if(0xfffd <= c && c <= 0xffff) { | |
| setParseError("string contains U+FFFD, U+FFFE or U+FFFF", errorCode); | |
| return i; | |
| } | |
| j += U16_LENGTH(c); | |
| } | |
| return i; | |
| } | |
| namespace { | |
| static const char *const positions[] = { | |
| "first tertiary ignorable", | |
| "last tertiary ignorable", | |
| "first secondary ignorable", | |
| "last secondary ignorable", | |
| "first primary ignorable", | |
| "last primary ignorable", | |
| "first variable", | |
| "last variable", | |
| "first regular", | |
| "last regular", | |
| "first implicit", | |
| "last implicit", | |
| "first trailing", | |
| "last trailing" | |
| }; | |
| } // namespace | |
| int32_t | |
| CollationRuleParser::parseSpecialPosition(int32_t i, UnicodeString &str, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return 0; } | |
| UnicodeString raw; | |
| int32_t j = readWords(i + 1, raw); | |
| if(j > i && rules->charAt(j) == 0x5d && !raw.isEmpty()) { // words end with ] | |
| ++j; | |
| for(int32_t pos = 0; pos < UPRV_LENGTHOF(positions); ++pos) { | |
| if(raw == UnicodeString(positions[pos], -1, US_INV)) { | |
| str.setTo((UChar)POS_LEAD).append((UChar)(POS_BASE + pos)); | |
| return j; | |
| } | |
| } | |
| if(raw == UNICODE_STRING_SIMPLE("top")) { | |
| str.setTo((UChar)POS_LEAD).append((UChar)(POS_BASE + LAST_REGULAR)); | |
| return j; | |
| } | |
| if(raw == UNICODE_STRING_SIMPLE("variable top")) { | |
| str.setTo((UChar)POS_LEAD).append((UChar)(POS_BASE + LAST_VARIABLE)); | |
| return j; | |
| } | |
| } | |
| setParseError("not a valid special reset position", errorCode); | |
| return i; | |
| } | |
| void | |
| CollationRuleParser::parseSetting(UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| UnicodeString raw; | |
| int32_t i = ruleIndex + 1; | |
| int32_t j = readWords(i, raw); | |
| if(j <= i || raw.isEmpty()) { | |
| setParseError("expected a setting/option at '['", errorCode); | |
| } | |
| if(rules->charAt(j) == 0x5d) { // words end with ] | |
| ++j; | |
| if(raw.startsWith(UNICODE_STRING_SIMPLE("reorder")) && | |
| (raw.length() == 7 || raw.charAt(7) == 0x20)) { | |
| parseReordering(raw, errorCode); | |
| ruleIndex = j; | |
| return; | |
| } | |
| if(raw == UNICODE_STRING_SIMPLE("backwards 2")) { | |
| settings->setFlag(CollationSettings::BACKWARD_SECONDARY, | |
| UCOL_ON, 0, errorCode); | |
| ruleIndex = j; | |
| return; | |
| } | |
| UnicodeString v; | |
| int32_t valueIndex = raw.lastIndexOf((UChar)0x20); | |
| if(valueIndex >= 0) { | |
| v.setTo(raw, valueIndex + 1); | |
| raw.truncate(valueIndex); | |
| } | |
| if(raw == UNICODE_STRING_SIMPLE("strength") && v.length() == 1) { | |
| int32_t value = UCOL_DEFAULT; | |
| UChar c = v.charAt(0); | |
| if(0x31 <= c && c <= 0x34) { // 1..4 | |
| value = UCOL_PRIMARY + (c - 0x31); | |
| } else if(c == 0x49) { // 'I' | |
| value = UCOL_IDENTICAL; | |
| } | |
| if(value != UCOL_DEFAULT) { | |
| settings->setStrength(value, 0, errorCode); | |
| ruleIndex = j; | |
| return; | |
| } | |
| } else if(raw == UNICODE_STRING_SIMPLE("alternate")) { | |
| UColAttributeValue value = UCOL_DEFAULT; | |
| if(v == UNICODE_STRING_SIMPLE("non-ignorable")) { | |
| value = UCOL_NON_IGNORABLE; | |
| } else if(v == UNICODE_STRING_SIMPLE("shifted")) { | |
| value = UCOL_SHIFTED; | |
| } | |
| if(value != UCOL_DEFAULT) { | |
| settings->setAlternateHandling(value, 0, errorCode); | |
| ruleIndex = j; | |
| return; | |
| } | |
| } else if(raw == UNICODE_STRING_SIMPLE("maxVariable")) { | |
| int32_t value = UCOL_DEFAULT; | |
| if(v == UNICODE_STRING_SIMPLE("space")) { | |
| value = CollationSettings::MAX_VAR_SPACE; | |
| } else if(v == UNICODE_STRING_SIMPLE("punct")) { | |
| value = CollationSettings::MAX_VAR_PUNCT; | |
| } else if(v == UNICODE_STRING_SIMPLE("symbol")) { | |
| value = CollationSettings::MAX_VAR_SYMBOL; | |
| } else if(v == UNICODE_STRING_SIMPLE("currency")) { | |
| value = CollationSettings::MAX_VAR_CURRENCY; | |
| } | |
| if(value != UCOL_DEFAULT) { | |
| settings->setMaxVariable(value, 0, errorCode); | |
| settings->variableTop = baseData->getLastPrimaryForGroup( | |
| UCOL_REORDER_CODE_FIRST + value); | |
| U_ASSERT(settings->variableTop != 0); | |
| ruleIndex = j; | |
| return; | |
| } | |
| } else if(raw == UNICODE_STRING_SIMPLE("caseFirst")) { | |
| UColAttributeValue value = UCOL_DEFAULT; | |
| if(v == UNICODE_STRING_SIMPLE("off")) { | |
| value = UCOL_OFF; | |
| } else if(v == UNICODE_STRING_SIMPLE("lower")) { | |
| value = UCOL_LOWER_FIRST; | |
| } else if(v == UNICODE_STRING_SIMPLE("upper")) { | |
| value = UCOL_UPPER_FIRST; | |
| } | |
| if(value != UCOL_DEFAULT) { | |
| settings->setCaseFirst(value, 0, errorCode); | |
| ruleIndex = j; | |
| return; | |
| } | |
| } else if(raw == UNICODE_STRING_SIMPLE("caseLevel")) { | |
| UColAttributeValue value = getOnOffValue(v); | |
| if(value != UCOL_DEFAULT) { | |
| settings->setFlag(CollationSettings::CASE_LEVEL, value, 0, errorCode); | |
| ruleIndex = j; | |
| return; | |
| } | |
| } else if(raw == UNICODE_STRING_SIMPLE("normalization")) { | |
| UColAttributeValue value = getOnOffValue(v); | |
| if(value != UCOL_DEFAULT) { | |
| settings->setFlag(CollationSettings::CHECK_FCD, value, 0, errorCode); | |
| ruleIndex = j; | |
| return; | |
| } | |
| } else if(raw == UNICODE_STRING_SIMPLE("numericOrdering")) { | |
| UColAttributeValue value = getOnOffValue(v); | |
| if(value != UCOL_DEFAULT) { | |
| settings->setFlag(CollationSettings::NUMERIC, value, 0, errorCode); | |
| ruleIndex = j; | |
| return; | |
| } | |
| } else if(raw == UNICODE_STRING_SIMPLE("hiraganaQ")) { | |
| UColAttributeValue value = getOnOffValue(v); | |
| if(value != UCOL_DEFAULT) { | |
| if(value == UCOL_ON) { | |
| setParseError("[hiraganaQ on] is not supported", errorCode); | |
| } | |
| ruleIndex = j; | |
| return; | |
| } | |
| } else if(raw == UNICODE_STRING_SIMPLE("import")) { | |
| CharString lang; | |
| lang.appendInvariantChars(v, errorCode); | |
| if(errorCode == U_MEMORY_ALLOCATION_ERROR) { return; } | |
| // BCP 47 language tag -> ICU locale ID | |
| char localeID[ULOC_FULLNAME_CAPACITY]; | |
| int32_t parsedLength; | |
| int32_t length = uloc_forLanguageTag(lang.data(), localeID, ULOC_FULLNAME_CAPACITY, | |
| &parsedLength, &errorCode); | |
| if(U_FAILURE(errorCode) || | |
| parsedLength != lang.length() || length >= ULOC_FULLNAME_CAPACITY) { | |
| errorCode = U_ZERO_ERROR; | |
| setParseError("expected language tag in [import langTag]", errorCode); | |
| return; | |
| } | |
| // localeID minus all keywords | |
| char baseID[ULOC_FULLNAME_CAPACITY]; | |
| length = uloc_getBaseName(localeID, baseID, ULOC_FULLNAME_CAPACITY, &errorCode); | |
| if(U_FAILURE(errorCode) || length >= ULOC_KEYWORDS_CAPACITY) { | |
| errorCode = U_ZERO_ERROR; | |
| setParseError("expected language tag in [import langTag]", errorCode); | |
| return; | |
| } | |
| if(length == 3 && uprv_memcmp(baseID, "und", 3) == 0) { | |
| uprv_strcpy(baseID, "root"); | |
| } | |
| // @collation=type, or length=0 if not specified | |
| char collationType[ULOC_KEYWORDS_CAPACITY]; | |
| length = uloc_getKeywordValue(localeID, "collation", | |
| collationType, ULOC_KEYWORDS_CAPACITY, | |
| &errorCode); | |
| if(U_FAILURE(errorCode) || length >= ULOC_KEYWORDS_CAPACITY) { | |
| errorCode = U_ZERO_ERROR; | |
| setParseError("expected language tag in [import langTag]", errorCode); | |
| return; | |
| } | |
| if(importer == NULL) { | |
| setParseError("[import langTag] is not supported", errorCode); | |
| } else { | |
| UnicodeString importedRules; | |
| importer->getRules(baseID, length > 0 ? collationType : "standard", | |
| importedRules, errorReason, errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| if(errorReason == NULL) { | |
| errorReason = "[import langTag] failed"; | |
| } | |
| setErrorContext(); | |
| return; | |
| } | |
| const UnicodeString *outerRules = rules; | |
| int32_t outerRuleIndex = ruleIndex; | |
| parse(importedRules, errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| if(parseError != NULL) { | |
| parseError->offset = outerRuleIndex; | |
| } | |
| } | |
| rules = outerRules; | |
| ruleIndex = j; | |
| } | |
| return; | |
| } | |
| } else if(rules->charAt(j) == 0x5b) { // words end with [ | |
| UnicodeSet set; | |
| j = parseUnicodeSet(j, set, errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| if(raw == UNICODE_STRING_SIMPLE("optimize")) { | |
| sink->optimize(set, errorReason, errorCode); | |
| if(U_FAILURE(errorCode)) { setErrorContext(); } | |
| ruleIndex = j; | |
| return; | |
| } else if(raw == UNICODE_STRING_SIMPLE("suppressContractions")) { | |
| sink->suppressContractions(set, errorReason, errorCode); | |
| if(U_FAILURE(errorCode)) { setErrorContext(); } | |
| ruleIndex = j; | |
| return; | |
| } | |
| } | |
| setParseError("not a valid setting/option", errorCode); | |
| } | |
| void | |
| CollationRuleParser::parseReordering(const UnicodeString &raw, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| int32_t i = 7; // after "reorder" | |
| if(i == raw.length()) { | |
| // empty [reorder] with no codes | |
| settings->resetReordering(); | |
| return; | |
| } | |
| // Parse the codes in [reorder aa bb cc]. | |
| UVector32 reorderCodes(errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| CharString word; | |
| while(i < raw.length()) { | |
| ++i; // skip the word-separating space | |
| int32_t limit = raw.indexOf((UChar)0x20, i); | |
| if(limit < 0) { limit = raw.length(); } | |
| word.clear().appendInvariantChars(raw.tempSubStringBetween(i, limit), errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| int32_t code = getReorderCode(word.data()); | |
| if(code < 0) { | |
| setParseError("unknown script or reorder code", errorCode); | |
| return; | |
| } | |
| reorderCodes.addElement(code, errorCode); | |
| if(U_FAILURE(errorCode)) { return; } | |
| i = limit; | |
| } | |
| settings->setReordering(*baseData, reorderCodes.getBuffer(), reorderCodes.size(), errorCode); | |
| } | |
| static const char *const gSpecialReorderCodes[] = { | |
| "space", "punct", "symbol", "currency", "digit" | |
| }; | |
| int32_t | |
| CollationRuleParser::getReorderCode(const char *word) { | |
| for(int32_t i = 0; i < UPRV_LENGTHOF(gSpecialReorderCodes); ++i) { | |
| if(uprv_stricmp(word, gSpecialReorderCodes[i]) == 0) { | |
| return UCOL_REORDER_CODE_FIRST + i; | |
| } | |
| } | |
| int32_t script = u_getPropertyValueEnum(UCHAR_SCRIPT, word); | |
| if(script >= 0) { | |
| return script; | |
| } | |
| if(uprv_stricmp(word, "others") == 0) { | |
| return UCOL_REORDER_CODE_OTHERS; // same as Zzzz = USCRIPT_UNKNOWN | |
| } | |
| return -1; | |
| } | |
| UColAttributeValue | |
| CollationRuleParser::getOnOffValue(const UnicodeString &s) { | |
| if(s == UNICODE_STRING_SIMPLE("on")) { | |
| return UCOL_ON; | |
| } else if(s == UNICODE_STRING_SIMPLE("off")) { | |
| return UCOL_OFF; | |
| } else { | |
| return UCOL_DEFAULT; | |
| } | |
| } | |
| int32_t | |
| CollationRuleParser::parseUnicodeSet(int32_t i, UnicodeSet &set, UErrorCode &errorCode) { | |
| // Collect a UnicodeSet pattern between a balanced pair of [brackets]. | |
| int32_t level = 0; | |
| int32_t j = i; | |
| for(;;) { | |
| if(j == rules->length()) { | |
| setParseError("unbalanced UnicodeSet pattern brackets", errorCode); | |
| return j; | |
| } | |
| UChar c = rules->charAt(j++); | |
| if(c == 0x5b) { // '[' | |
| ++level; | |
| } else if(c == 0x5d) { // ']' | |
| if(--level == 0) { break; } | |
| } | |
| } | |
| set.applyPattern(rules->tempSubStringBetween(i, j), errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| errorCode = U_ZERO_ERROR; | |
| setParseError("not a valid UnicodeSet pattern", errorCode); | |
| return j; | |
| } | |
| j = skipWhiteSpace(j); | |
| if(j == rules->length() || rules->charAt(j) != 0x5d) { | |
| setParseError("missing option-terminating ']' after UnicodeSet pattern", errorCode); | |
| return j; | |
| } | |
| return ++j; | |
| } | |
| int32_t | |
| CollationRuleParser::readWords(int32_t i, UnicodeString &raw) const { | |
| static const UChar sp = 0x20; | |
| raw.remove(); | |
| i = skipWhiteSpace(i); | |
| for(;;) { | |
| if(i >= rules->length()) { return 0; } | |
| UChar c = rules->charAt(i); | |
| if(isSyntaxChar(c) && c != 0x2d && c != 0x5f) { // syntax except -_ | |
| if(raw.isEmpty()) { return i; } | |
| if(raw.endsWith(&sp, 1)) { // remove trailing space | |
| raw.truncate(raw.length() - 1); | |
| } | |
| return i; | |
| } | |
| if(PatternProps::isWhiteSpace(c)) { | |
| raw.append(0x20); | |
| i = skipWhiteSpace(i + 1); | |
| } else { | |
| raw.append(c); | |
| ++i; | |
| } | |
| } | |
| } | |
| int32_t | |
| CollationRuleParser::skipComment(int32_t i) const { | |
| // skip to past the newline | |
| while(i < rules->length()) { | |
| UChar c = rules->charAt(i++); | |
| // LF or FF or CR or NEL or LS or PS | |
| if(c == 0xa || c == 0xc || c == 0xd || c == 0x85 || c == 0x2028 || c == 0x2029) { | |
| // Unicode Newline Guidelines: "A readline function should stop at NLF, LS, FF, or PS." | |
| // NLF (new line function) = CR or LF or CR+LF or NEL. | |
| // No need to collect all of CR+LF because a following LF will be ignored anyway. | |
| break; | |
| } | |
| } | |
| return i; | |
| } | |
| void | |
| CollationRuleParser::setParseError(const char *reason, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| // Error code consistent with the old parser (from ca. 2001), | |
| // rather than U_PARSE_ERROR; | |
| errorCode = U_INVALID_FORMAT_ERROR; | |
| errorReason = reason; | |
| if(parseError != NULL) { setErrorContext(); } | |
| } | |
| void | |
| CollationRuleParser::setErrorContext() { | |
| if(parseError == NULL) { return; } | |
| // Note: This relies on the calling code maintaining the ruleIndex | |
| // at a position that is useful for debugging. | |
| // For example, at the beginning of a reset or relation etc. | |
| parseError->offset = ruleIndex; | |
| parseError->line = 0; // We are not counting line numbers. | |
| // before ruleIndex | |
| int32_t start = ruleIndex - (U_PARSE_CONTEXT_LEN - 1); | |
| if(start < 0) { | |
| start = 0; | |
| } else if(start > 0 && U16_IS_TRAIL(rules->charAt(start))) { | |
| ++start; | |
| } | |
| int32_t length = ruleIndex - start; | |
| rules->extract(start, length, parseError->preContext); | |
| parseError->preContext[length] = 0; | |
| // starting from ruleIndex | |
| length = rules->length() - ruleIndex; | |
| if(length >= U_PARSE_CONTEXT_LEN) { | |
| length = U_PARSE_CONTEXT_LEN - 1; | |
| if(U16_IS_LEAD(rules->charAt(ruleIndex + length - 1))) { | |
| --length; | |
| } | |
| } | |
| rules->extract(ruleIndex, length, parseError->postContext); | |
| parseError->postContext[length] = 0; | |
| } | |
| UBool | |
| CollationRuleParser::isSyntaxChar(UChar32 c) { | |
| return 0x21 <= c && c <= 0x7e && | |
| (c <= 0x2f || (0x3a <= c && c <= 0x40) || | |
| (0x5b <= c && c <= 0x60) || (0x7b <= c)); | |
| } | |
| int32_t | |
| CollationRuleParser::skipWhiteSpace(int32_t i) const { | |
| while(i < rules->length() && PatternProps::isWhiteSpace(rules->charAt(i))) { | |
| ++i; | |
| } | |
| return i; | |
| } | |
| U_NAMESPACE_END | |
| #endif // !UCONFIG_NO_COLLATION |