| /* |
| ******************************************************************************* |
| * Copyright (C) 2013-2015, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| ******************************************************************************* |
| * collationruleparser.cpp |
| * |
| * (replaced the former ucol_tok.cpp) |
| * |
| * created on: 2013apr10 |
| * created by: Markus W. Scherer |
| */ |
| |
| #include "unicode/utypes.h" |
| |
| #if !UCONFIG_NO_COLLATION |
| |
| #include "starboard/client_porting/poem/assert_poem.h" |
| #include "starboard/client_porting/poem/string_poem.h" |
| #include "unicode/normalizer2.h" |
| #include "unicode/parseerr.h" |
| #include "unicode/uchar.h" |
| #include "unicode/ucol.h" |
| #include "unicode/uloc.h" |
| #include "unicode/unistr.h" |
| #include "unicode/utf16.h" |
| #include "charstr.h" |
| #include "cmemory.h" |
| #include "collation.h" |
| #include "collationdata.h" |
| #include "collationruleparser.h" |
| #include "collationsettings.h" |
| #include "collationtailoring.h" |
| #include "cstring.h" |
| #include "patternprops.h" |
| #include "uassert.h" |
| #include "uvectr32.h" |
| |
| U_NAMESPACE_BEGIN |
| |
| namespace { |
| |
| static const UChar BEFORE[] = { 0x5b, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, 0 }; // "[before" |
| const int32_t BEFORE_LENGTH = 7; |
| |
| } // namespace |
| |
| CollationRuleParser::Sink::~Sink() {} |
| |
| void |
| CollationRuleParser::Sink::suppressContractions(const UnicodeSet &, const char *&, UErrorCode &) {} |
| |
| void |
| CollationRuleParser::Sink::optimize(const UnicodeSet &, const char *&, UErrorCode &) {} |
| |
| CollationRuleParser::Importer::~Importer() {} |
| |
| CollationRuleParser::CollationRuleParser(const CollationData *base, UErrorCode &errorCode) |
| : nfd(*Normalizer2::getNFDInstance(errorCode)), |
| nfc(*Normalizer2::getNFCInstance(errorCode)), |
| rules(NULL), baseData(base), settings(NULL), |
| parseError(NULL), errorReason(NULL), |
| sink(NULL), importer(NULL), |
| ruleIndex(0) { |
| } |
| |
| CollationRuleParser::~CollationRuleParser() { |
| } |
| |
| void |
| CollationRuleParser::parse(const UnicodeString &ruleString, |
| CollationSettings &outSettings, |
| UParseError *outParseError, |
| UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode)) { return; } |
| settings = &outSettings; |
| parseError = outParseError; |
| if(parseError != NULL) { |
| parseError->line = 0; |
| parseError->offset = -1; |
| parseError->preContext[0] = 0; |
| parseError->postContext[0] = 0; |
| } |
| errorReason = NULL; |
| parse(ruleString, errorCode); |
| } |
| |
| void |
| CollationRuleParser::parse(const UnicodeString &ruleString, UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode)) { return; } |
| rules = &ruleString; |
| ruleIndex = 0; |
| |
| while(ruleIndex < rules->length()) { |
| UChar c = rules->charAt(ruleIndex); |
| if(PatternProps::isWhiteSpace(c)) { |
| ++ruleIndex; |
| continue; |
| } |
| switch(c) { |
| case 0x26: // '&' |
| parseRuleChain(errorCode); |
| break; |
| case 0x5b: // '[' |
| parseSetting(errorCode); |
| break; |
| case 0x23: // '#' starts a comment, until the end of the line |
| ruleIndex = skipComment(ruleIndex + 1); |
| break; |
| case 0x40: // '@' is equivalent to [backwards 2] |
| settings->setFlag(CollationSettings::BACKWARD_SECONDARY, |
| UCOL_ON, 0, errorCode); |
| ++ruleIndex; |
| break; |
| case 0x21: // '!' used to turn on Thai/Lao character reversal |
| // Accept but ignore. The root collator has contractions |
| // that are equivalent to the character reversal, where appropriate. |
| ++ruleIndex; |
| break; |
| default: |
| setParseError("expected a reset or setting or comment", errorCode); |
| break; |
| } |
| if(U_FAILURE(errorCode)) { return; } |
| } |
| } |
| |
| void |
| CollationRuleParser::parseRuleChain(UErrorCode &errorCode) { |
| int32_t resetStrength = parseResetAndPosition(errorCode); |
| UBool isFirstRelation = TRUE; |
| for(;;) { |
| int32_t result = parseRelationOperator(errorCode); |
| if(U_FAILURE(errorCode)) { return; } |
| if(result < 0) { |
| if(ruleIndex < rules->length() && rules->charAt(ruleIndex) == 0x23) { |
| // '#' starts a comment, until the end of the line |
| ruleIndex = skipComment(ruleIndex + 1); |
| continue; |
| } |
| if(isFirstRelation) { |
| setParseError("reset not followed by a relation", errorCode); |
| } |
| return; |
| } |
| int32_t strength = result & STRENGTH_MASK; |
| if(resetStrength < UCOL_IDENTICAL) { |
| // reset-before rule chain |
| if(isFirstRelation) { |
| if(strength != resetStrength) { |
| setParseError("reset-before strength differs from its first relation", errorCode); |
| return; |
| } |
| } else { |
| if(strength < resetStrength) { |
| setParseError("reset-before strength followed by a stronger relation", errorCode); |
| return; |
| } |
| } |
| } |
| int32_t i = ruleIndex + (result >> OFFSET_SHIFT); // skip over the relation operator |
| if((result & STARRED_FLAG) == 0) { |
| parseRelationStrings(strength, i, errorCode); |
| } else { |
| parseStarredCharacters(strength, i, errorCode); |
| } |
| if(U_FAILURE(errorCode)) { return; } |
| isFirstRelation = FALSE; |
| } |
| } |
| |
| int32_t |
| CollationRuleParser::parseResetAndPosition(UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode)) { return UCOL_DEFAULT; } |
| int32_t i = skipWhiteSpace(ruleIndex + 1); |
| int32_t j; |
| UChar c; |
| int32_t resetStrength; |
| if(rules->compare(i, BEFORE_LENGTH, BEFORE, 0, BEFORE_LENGTH) == 0 && |
| (j = i + BEFORE_LENGTH) < rules->length() && |
| PatternProps::isWhiteSpace(rules->charAt(j)) && |
| ((j = skipWhiteSpace(j + 1)) + 1) < rules->length() && |
| 0x31 <= (c = rules->charAt(j)) && c <= 0x33 && |
| rules->charAt(j + 1) == 0x5d) { |
| // &[before n] with n=1 or 2 or 3 |
| resetStrength = UCOL_PRIMARY + (c - 0x31); |
| i = skipWhiteSpace(j + 2); |
| } else { |
| resetStrength = UCOL_IDENTICAL; |
| } |
| if(i >= rules->length()) { |
| setParseError("reset without position", errorCode); |
| return UCOL_DEFAULT; |
| } |
| UnicodeString str; |
| if(rules->charAt(i) == 0x5b) { // '[' |
| i = parseSpecialPosition(i, str, errorCode); |
| } else { |
| i = parseTailoringString(i, str, errorCode); |
| } |
| sink->addReset(resetStrength, str, errorReason, errorCode); |
| if(U_FAILURE(errorCode)) { setErrorContext(); } |
| ruleIndex = i; |
| return resetStrength; |
| } |
| |
| int32_t |
| CollationRuleParser::parseRelationOperator(UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode)) { return UCOL_DEFAULT; } |
| ruleIndex = skipWhiteSpace(ruleIndex); |
| if(ruleIndex >= rules->length()) { return UCOL_DEFAULT; } |
| int32_t strength; |
| int32_t i = ruleIndex; |
| UChar c = rules->charAt(i++); |
| switch(c) { |
| case 0x3c: // '<' |
| if(i < rules->length() && rules->charAt(i) == 0x3c) { // << |
| ++i; |
| if(i < rules->length() && rules->charAt(i) == 0x3c) { // <<< |
| ++i; |
| if(i < rules->length() && rules->charAt(i) == 0x3c) { // <<<< |
| ++i; |
| strength = UCOL_QUATERNARY; |
| } else { |
| strength = UCOL_TERTIARY; |
| } |
| } else { |
| strength = UCOL_SECONDARY; |
| } |
| } else { |
| strength = UCOL_PRIMARY; |
| } |
| if(i < rules->length() && rules->charAt(i) == 0x2a) { // '*' |
| ++i; |
| strength |= STARRED_FLAG; |
| } |
| break; |
| case 0x3b: // ';' same as << |
| strength = UCOL_SECONDARY; |
| break; |
| case 0x2c: // ',' same as <<< |
| strength = UCOL_TERTIARY; |
| break; |
| case 0x3d: // '=' |
| strength = UCOL_IDENTICAL; |
| if(i < rules->length() && rules->charAt(i) == 0x2a) { // '*' |
| ++i; |
| strength |= STARRED_FLAG; |
| } |
| break; |
| default: |
| return UCOL_DEFAULT; |
| } |
| return ((i - ruleIndex) << OFFSET_SHIFT) | strength; |
| } |
| |
| void |
| CollationRuleParser::parseRelationStrings(int32_t strength, int32_t i, UErrorCode &errorCode) { |
| // Parse |
| // prefix | str / extension |
| // where prefix and extension are optional. |
| UnicodeString prefix, str, extension; |
| i = parseTailoringString(i, str, errorCode); |
| if(U_FAILURE(errorCode)) { return; } |
| UChar next = (i < rules->length()) ? rules->charAt(i) : 0; |
| if(next == 0x7c) { // '|' separates the context prefix from the string. |
| prefix = str; |
| i = parseTailoringString(i + 1, str, errorCode); |
| if(U_FAILURE(errorCode)) { return; } |
| next = (i < rules->length()) ? rules->charAt(i) : 0; |
| } |
| if(next == 0x2f) { // '/' separates the string from the extension. |
| i = parseTailoringString(i + 1, extension, errorCode); |
| } |
| if(!prefix.isEmpty()) { |
| UChar32 prefix0 = prefix.char32At(0); |
| UChar32 c = str.char32At(0); |
| if(!nfc.hasBoundaryBefore(prefix0) || !nfc.hasBoundaryBefore(c)) { |
| setParseError("in 'prefix|str', prefix and str must each start with an NFC boundary", |
| errorCode); |
| return; |
| } |
| } |
| sink->addRelation(strength, prefix, str, extension, errorReason, errorCode); |
| if(U_FAILURE(errorCode)) { setErrorContext(); } |
| ruleIndex = i; |
| } |
| |
| void |
| CollationRuleParser::parseStarredCharacters(int32_t strength, int32_t i, UErrorCode &errorCode) { |
| UnicodeString empty, raw; |
| i = parseString(skipWhiteSpace(i), raw, errorCode); |
| if(U_FAILURE(errorCode)) { return; } |
| if(raw.isEmpty()) { |
| setParseError("missing starred-relation string", errorCode); |
| return; |
| } |
| UChar32 prev = -1; |
| int32_t j = 0; |
| for(;;) { |
| while(j < raw.length()) { |
| UChar32 c = raw.char32At(j); |
| if(!nfd.isInert(c)) { |
| setParseError("starred-relation string is not all NFD-inert", errorCode); |
| return; |
| } |
| sink->addRelation(strength, empty, UnicodeString(c), empty, errorReason, errorCode); |
| if(U_FAILURE(errorCode)) { |
| setErrorContext(); |
| return; |
| } |
| j += U16_LENGTH(c); |
| prev = c; |
| } |
| if(i >= rules->length() || rules->charAt(i) != 0x2d) { // '-' |
| break; |
| } |
| if(prev < 0) { |
| setParseError("range without start in starred-relation string", errorCode); |
| return; |
| } |
| i = parseString(i + 1, raw, errorCode); |
| if(U_FAILURE(errorCode)) { return; } |
| if(raw.isEmpty()) { |
| setParseError("range without end in starred-relation string", errorCode); |
| return; |
| } |
| UChar32 c = raw.char32At(0); |
| if(c < prev) { |
| setParseError("range start greater than end in starred-relation string", errorCode); |
| return; |
| } |
| // range prev-c |
| UnicodeString s; |
| while(++prev <= c) { |
| if(!nfd.isInert(prev)) { |
| setParseError("starred-relation string range is not all NFD-inert", errorCode); |
| return; |
| } |
| if(U_IS_SURROGATE(prev)) { |
| setParseError("starred-relation string range contains a surrogate", errorCode); |
| return; |
| } |
| if(0xfffd <= prev && prev <= 0xffff) { |
| setParseError("starred-relation string range contains U+FFFD, U+FFFE or U+FFFF", errorCode); |
| return; |
| } |
| s.setTo(prev); |
| sink->addRelation(strength, empty, s, empty, errorReason, errorCode); |
| if(U_FAILURE(errorCode)) { |
| setErrorContext(); |
| return; |
| } |
| } |
| prev = -1; |
| j = U16_LENGTH(c); |
| } |
| ruleIndex = skipWhiteSpace(i); |
| } |
| |
| int32_t |
| CollationRuleParser::parseTailoringString(int32_t i, UnicodeString &raw, UErrorCode &errorCode) { |
| i = parseString(skipWhiteSpace(i), raw, errorCode); |
| if(U_SUCCESS(errorCode) && raw.isEmpty()) { |
| setParseError("missing relation string", errorCode); |
| } |
| return skipWhiteSpace(i); |
| } |
| |
| int32_t |
| CollationRuleParser::parseString(int32_t i, UnicodeString &raw, UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode)) { return i; } |
| raw.remove(); |
| while(i < rules->length()) { |
| UChar32 c = rules->charAt(i++); |
| if(isSyntaxChar(c)) { |
| if(c == 0x27) { // apostrophe |
| if(i < rules->length() && rules->charAt(i) == 0x27) { |
| // Double apostrophe, encodes a single one. |
| raw.append((UChar)0x27); |
| ++i; |
| continue; |
| } |
| // Quote literal text until the next single apostrophe. |
| for(;;) { |
| if(i == rules->length()) { |
| setParseError("quoted literal text missing terminating apostrophe", errorCode); |
| return i; |
| } |
| c = rules->charAt(i++); |
| if(c == 0x27) { |
| if(i < rules->length() && rules->charAt(i) == 0x27) { |
| // Double apostrophe inside quoted literal text, |
| // still encodes a single apostrophe. |
| ++i; |
| } else { |
| break; |
| } |
| } |
| raw.append((UChar)c); |
| } |
| } else if(c == 0x5c) { // backslash |
| if(i == rules->length()) { |
| setParseError("backslash escape at the end of the rule string", errorCode); |
| return i; |
| } |
| c = rules->char32At(i); |
| raw.append(c); |
| i += U16_LENGTH(c); |
| } else { |
| // Any other syntax character terminates a string. |
| --i; |
| break; |
| } |
| } else if(PatternProps::isWhiteSpace(c)) { |
| // Unquoted white space terminates a string. |
| --i; |
| break; |
| } else { |
| raw.append((UChar)c); |
| } |
| } |
| for(int32_t j = 0; j < raw.length();) { |
| UChar32 c = raw.char32At(j); |
| if(U_IS_SURROGATE(c)) { |
| setParseError("string contains an unpaired surrogate", errorCode); |
| return i; |
| } |
| if(0xfffd <= c && c <= 0xffff) { |
| setParseError("string contains U+FFFD, U+FFFE or U+FFFF", errorCode); |
| return i; |
| } |
| j += U16_LENGTH(c); |
| } |
| return i; |
| } |
| |
| namespace { |
| |
| static const char *const positions[] = { |
| "first tertiary ignorable", |
| "last tertiary ignorable", |
| "first secondary ignorable", |
| "last secondary ignorable", |
| "first primary ignorable", |
| "last primary ignorable", |
| "first variable", |
| "last variable", |
| "first regular", |
| "last regular", |
| "first implicit", |
| "last implicit", |
| "first trailing", |
| "last trailing" |
| }; |
| |
| } // namespace |
| |
| int32_t |
| CollationRuleParser::parseSpecialPosition(int32_t i, UnicodeString &str, UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode)) { return 0; } |
| UnicodeString raw; |
| int32_t j = readWords(i + 1, raw); |
| if(j > i && rules->charAt(j) == 0x5d && !raw.isEmpty()) { // words end with ] |
| ++j; |
| for(int32_t pos = 0; pos < UPRV_LENGTHOF(positions); ++pos) { |
| if(raw == UnicodeString(positions[pos], -1, US_INV)) { |
| str.setTo((UChar)POS_LEAD).append((UChar)(POS_BASE + pos)); |
| return j; |
| } |
| } |
| if(raw == UNICODE_STRING_SIMPLE("top")) { |
| str.setTo((UChar)POS_LEAD).append((UChar)(POS_BASE + LAST_REGULAR)); |
| return j; |
| } |
| if(raw == UNICODE_STRING_SIMPLE("variable top")) { |
| str.setTo((UChar)POS_LEAD).append((UChar)(POS_BASE + LAST_VARIABLE)); |
| return j; |
| } |
| } |
| setParseError("not a valid special reset position", errorCode); |
| return i; |
| } |
| |
| void |
| CollationRuleParser::parseSetting(UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode)) { return; } |
| UnicodeString raw; |
| int32_t i = ruleIndex + 1; |
| int32_t j = readWords(i, raw); |
| if(j <= i || raw.isEmpty()) { |
| setParseError("expected a setting/option at '['", errorCode); |
| } |
| if(rules->charAt(j) == 0x5d) { // words end with ] |
| ++j; |
| if(raw.startsWith(UNICODE_STRING_SIMPLE("reorder")) && |
| (raw.length() == 7 || raw.charAt(7) == 0x20)) { |
| parseReordering(raw, errorCode); |
| ruleIndex = j; |
| return; |
| } |
| if(raw == UNICODE_STRING_SIMPLE("backwards 2")) { |
| settings->setFlag(CollationSettings::BACKWARD_SECONDARY, |
| UCOL_ON, 0, errorCode); |
| ruleIndex = j; |
| return; |
| } |
| UnicodeString v; |
| int32_t valueIndex = raw.lastIndexOf((UChar)0x20); |
| if(valueIndex >= 0) { |
| v.setTo(raw, valueIndex + 1); |
| raw.truncate(valueIndex); |
| } |
| if(raw == UNICODE_STRING_SIMPLE("strength") && v.length() == 1) { |
| int32_t value = UCOL_DEFAULT; |
| UChar c = v.charAt(0); |
| if(0x31 <= c && c <= 0x34) { // 1..4 |
| value = UCOL_PRIMARY + (c - 0x31); |
| } else if(c == 0x49) { // 'I' |
| value = UCOL_IDENTICAL; |
| } |
| if(value != UCOL_DEFAULT) { |
| settings->setStrength(value, 0, errorCode); |
| ruleIndex = j; |
| return; |
| } |
| } else if(raw == UNICODE_STRING_SIMPLE("alternate")) { |
| UColAttributeValue value = UCOL_DEFAULT; |
| if(v == UNICODE_STRING_SIMPLE("non-ignorable")) { |
| value = UCOL_NON_IGNORABLE; |
| } else if(v == UNICODE_STRING_SIMPLE("shifted")) { |
| value = UCOL_SHIFTED; |
| } |
| if(value != UCOL_DEFAULT) { |
| settings->setAlternateHandling(value, 0, errorCode); |
| ruleIndex = j; |
| return; |
| } |
| } else if(raw == UNICODE_STRING_SIMPLE("maxVariable")) { |
| int32_t value = UCOL_DEFAULT; |
| if(v == UNICODE_STRING_SIMPLE("space")) { |
| value = CollationSettings::MAX_VAR_SPACE; |
| } else if(v == UNICODE_STRING_SIMPLE("punct")) { |
| value = CollationSettings::MAX_VAR_PUNCT; |
| } else if(v == UNICODE_STRING_SIMPLE("symbol")) { |
| value = CollationSettings::MAX_VAR_SYMBOL; |
| } else if(v == UNICODE_STRING_SIMPLE("currency")) { |
| value = CollationSettings::MAX_VAR_CURRENCY; |
| } |
| if(value != UCOL_DEFAULT) { |
| settings->setMaxVariable(value, 0, errorCode); |
| settings->variableTop = baseData->getLastPrimaryForGroup( |
| UCOL_REORDER_CODE_FIRST + value); |
| U_ASSERT(settings->variableTop != 0); |
| ruleIndex = j; |
| return; |
| } |
| } else if(raw == UNICODE_STRING_SIMPLE("caseFirst")) { |
| UColAttributeValue value = UCOL_DEFAULT; |
| if(v == UNICODE_STRING_SIMPLE("off")) { |
| value = UCOL_OFF; |
| } else if(v == UNICODE_STRING_SIMPLE("lower")) { |
| value = UCOL_LOWER_FIRST; |
| } else if(v == UNICODE_STRING_SIMPLE("upper")) { |
| value = UCOL_UPPER_FIRST; |
| } |
| if(value != UCOL_DEFAULT) { |
| settings->setCaseFirst(value, 0, errorCode); |
| ruleIndex = j; |
| return; |
| } |
| } else if(raw == UNICODE_STRING_SIMPLE("caseLevel")) { |
| UColAttributeValue value = getOnOffValue(v); |
| if(value != UCOL_DEFAULT) { |
| settings->setFlag(CollationSettings::CASE_LEVEL, value, 0, errorCode); |
| ruleIndex = j; |
| return; |
| } |
| } else if(raw == UNICODE_STRING_SIMPLE("normalization")) { |
| UColAttributeValue value = getOnOffValue(v); |
| if(value != UCOL_DEFAULT) { |
| settings->setFlag(CollationSettings::CHECK_FCD, value, 0, errorCode); |
| ruleIndex = j; |
| return; |
| } |
| } else if(raw == UNICODE_STRING_SIMPLE("numericOrdering")) { |
| UColAttributeValue value = getOnOffValue(v); |
| if(value != UCOL_DEFAULT) { |
| settings->setFlag(CollationSettings::NUMERIC, value, 0, errorCode); |
| ruleIndex = j; |
| return; |
| } |
| } else if(raw == UNICODE_STRING_SIMPLE("hiraganaQ")) { |
| UColAttributeValue value = getOnOffValue(v); |
| if(value != UCOL_DEFAULT) { |
| if(value == UCOL_ON) { |
| setParseError("[hiraganaQ on] is not supported", errorCode); |
| } |
| ruleIndex = j; |
| return; |
| } |
| } else if(raw == UNICODE_STRING_SIMPLE("import")) { |
| CharString lang; |
| lang.appendInvariantChars(v, errorCode); |
| if(errorCode == U_MEMORY_ALLOCATION_ERROR) { return; } |
| // BCP 47 language tag -> ICU locale ID |
| char localeID[ULOC_FULLNAME_CAPACITY]; |
| int32_t parsedLength; |
| int32_t length = uloc_forLanguageTag(lang.data(), localeID, ULOC_FULLNAME_CAPACITY, |
| &parsedLength, &errorCode); |
| if(U_FAILURE(errorCode) || |
| parsedLength != lang.length() || length >= ULOC_FULLNAME_CAPACITY) { |
| errorCode = U_ZERO_ERROR; |
| setParseError("expected language tag in [import langTag]", errorCode); |
| return; |
| } |
| // localeID minus all keywords |
| char baseID[ULOC_FULLNAME_CAPACITY]; |
| length = uloc_getBaseName(localeID, baseID, ULOC_FULLNAME_CAPACITY, &errorCode); |
| if(U_FAILURE(errorCode) || length >= ULOC_KEYWORDS_CAPACITY) { |
| errorCode = U_ZERO_ERROR; |
| setParseError("expected language tag in [import langTag]", errorCode); |
| return; |
| } |
| if(length == 3 && uprv_memcmp(baseID, "und", 3) == 0) { |
| uprv_strcpy(baseID, "root"); |
| } |
| // @collation=type, or length=0 if not specified |
| char collationType[ULOC_KEYWORDS_CAPACITY]; |
| length = uloc_getKeywordValue(localeID, "collation", |
| collationType, ULOC_KEYWORDS_CAPACITY, |
| &errorCode); |
| if(U_FAILURE(errorCode) || length >= ULOC_KEYWORDS_CAPACITY) { |
| errorCode = U_ZERO_ERROR; |
| setParseError("expected language tag in [import langTag]", errorCode); |
| return; |
| } |
| if(importer == NULL) { |
| setParseError("[import langTag] is not supported", errorCode); |
| } else { |
| UnicodeString importedRules; |
| importer->getRules(baseID, length > 0 ? collationType : "standard", |
| importedRules, errorReason, errorCode); |
| if(U_FAILURE(errorCode)) { |
| if(errorReason == NULL) { |
| errorReason = "[import langTag] failed"; |
| } |
| setErrorContext(); |
| return; |
| } |
| const UnicodeString *outerRules = rules; |
| int32_t outerRuleIndex = ruleIndex; |
| parse(importedRules, errorCode); |
| if(U_FAILURE(errorCode)) { |
| if(parseError != NULL) { |
| parseError->offset = outerRuleIndex; |
| } |
| } |
| rules = outerRules; |
| ruleIndex = j; |
| } |
| return; |
| } |
| } else if(rules->charAt(j) == 0x5b) { // words end with [ |
| UnicodeSet set; |
| j = parseUnicodeSet(j, set, errorCode); |
| if(U_FAILURE(errorCode)) { return; } |
| if(raw == UNICODE_STRING_SIMPLE("optimize")) { |
| sink->optimize(set, errorReason, errorCode); |
| if(U_FAILURE(errorCode)) { setErrorContext(); } |
| ruleIndex = j; |
| return; |
| } else if(raw == UNICODE_STRING_SIMPLE("suppressContractions")) { |
| sink->suppressContractions(set, errorReason, errorCode); |
| if(U_FAILURE(errorCode)) { setErrorContext(); } |
| ruleIndex = j; |
| return; |
| } |
| } |
| setParseError("not a valid setting/option", errorCode); |
| } |
| |
| void |
| CollationRuleParser::parseReordering(const UnicodeString &raw, UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode)) { return; } |
| int32_t i = 7; // after "reorder" |
| if(i == raw.length()) { |
| // empty [reorder] with no codes |
| settings->resetReordering(); |
| return; |
| } |
| // Parse the codes in [reorder aa bb cc]. |
| UVector32 reorderCodes(errorCode); |
| if(U_FAILURE(errorCode)) { return; } |
| CharString word; |
| while(i < raw.length()) { |
| ++i; // skip the word-separating space |
| int32_t limit = raw.indexOf((UChar)0x20, i); |
| if(limit < 0) { limit = raw.length(); } |
| word.clear().appendInvariantChars(raw.tempSubStringBetween(i, limit), errorCode); |
| if(U_FAILURE(errorCode)) { return; } |
| int32_t code = getReorderCode(word.data()); |
| if(code < 0) { |
| setParseError("unknown script or reorder code", errorCode); |
| return; |
| } |
| reorderCodes.addElement(code, errorCode); |
| if(U_FAILURE(errorCode)) { return; } |
| i = limit; |
| } |
| settings->setReordering(*baseData, reorderCodes.getBuffer(), reorderCodes.size(), errorCode); |
| } |
| |
| static const char *const gSpecialReorderCodes[] = { |
| "space", "punct", "symbol", "currency", "digit" |
| }; |
| |
| int32_t |
| CollationRuleParser::getReorderCode(const char *word) { |
| for(int32_t i = 0; i < UPRV_LENGTHOF(gSpecialReorderCodes); ++i) { |
| if(uprv_stricmp(word, gSpecialReorderCodes[i]) == 0) { |
| return UCOL_REORDER_CODE_FIRST + i; |
| } |
| } |
| int32_t script = u_getPropertyValueEnum(UCHAR_SCRIPT, word); |
| if(script >= 0) { |
| return script; |
| } |
| if(uprv_stricmp(word, "others") == 0) { |
| return UCOL_REORDER_CODE_OTHERS; // same as Zzzz = USCRIPT_UNKNOWN |
| } |
| return -1; |
| } |
| |
| UColAttributeValue |
| CollationRuleParser::getOnOffValue(const UnicodeString &s) { |
| if(s == UNICODE_STRING_SIMPLE("on")) { |
| return UCOL_ON; |
| } else if(s == UNICODE_STRING_SIMPLE("off")) { |
| return UCOL_OFF; |
| } else { |
| return UCOL_DEFAULT; |
| } |
| } |
| |
| int32_t |
| CollationRuleParser::parseUnicodeSet(int32_t i, UnicodeSet &set, UErrorCode &errorCode) { |
| // Collect a UnicodeSet pattern between a balanced pair of [brackets]. |
| int32_t level = 0; |
| int32_t j = i; |
| for(;;) { |
| if(j == rules->length()) { |
| setParseError("unbalanced UnicodeSet pattern brackets", errorCode); |
| return j; |
| } |
| UChar c = rules->charAt(j++); |
| if(c == 0x5b) { // '[' |
| ++level; |
| } else if(c == 0x5d) { // ']' |
| if(--level == 0) { break; } |
| } |
| } |
| set.applyPattern(rules->tempSubStringBetween(i, j), errorCode); |
| if(U_FAILURE(errorCode)) { |
| errorCode = U_ZERO_ERROR; |
| setParseError("not a valid UnicodeSet pattern", errorCode); |
| return j; |
| } |
| j = skipWhiteSpace(j); |
| if(j == rules->length() || rules->charAt(j) != 0x5d) { |
| setParseError("missing option-terminating ']' after UnicodeSet pattern", errorCode); |
| return j; |
| } |
| return ++j; |
| } |
| |
| int32_t |
| CollationRuleParser::readWords(int32_t i, UnicodeString &raw) const { |
| static const UChar sp = 0x20; |
| raw.remove(); |
| i = skipWhiteSpace(i); |
| for(;;) { |
| if(i >= rules->length()) { return 0; } |
| UChar c = rules->charAt(i); |
| if(isSyntaxChar(c) && c != 0x2d && c != 0x5f) { // syntax except -_ |
| if(raw.isEmpty()) { return i; } |
| if(raw.endsWith(&sp, 1)) { // remove trailing space |
| raw.truncate(raw.length() - 1); |
| } |
| return i; |
| } |
| if(PatternProps::isWhiteSpace(c)) { |
| raw.append(0x20); |
| i = skipWhiteSpace(i + 1); |
| } else { |
| raw.append(c); |
| ++i; |
| } |
| } |
| } |
| |
| int32_t |
| CollationRuleParser::skipComment(int32_t i) const { |
| // skip to past the newline |
| while(i < rules->length()) { |
| UChar c = rules->charAt(i++); |
| // LF or FF or CR or NEL or LS or PS |
| if(c == 0xa || c == 0xc || c == 0xd || c == 0x85 || c == 0x2028 || c == 0x2029) { |
| // Unicode Newline Guidelines: "A readline function should stop at NLF, LS, FF, or PS." |
| // NLF (new line function) = CR or LF or CR+LF or NEL. |
| // No need to collect all of CR+LF because a following LF will be ignored anyway. |
| break; |
| } |
| } |
| return i; |
| } |
| |
| void |
| CollationRuleParser::setParseError(const char *reason, UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode)) { return; } |
| // Error code consistent with the old parser (from ca. 2001), |
| // rather than U_PARSE_ERROR; |
| errorCode = U_INVALID_FORMAT_ERROR; |
| errorReason = reason; |
| if(parseError != NULL) { setErrorContext(); } |
| } |
| |
| void |
| CollationRuleParser::setErrorContext() { |
| if(parseError == NULL) { return; } |
| |
| // Note: This relies on the calling code maintaining the ruleIndex |
| // at a position that is useful for debugging. |
| // For example, at the beginning of a reset or relation etc. |
| parseError->offset = ruleIndex; |
| parseError->line = 0; // We are not counting line numbers. |
| |
| // before ruleIndex |
| int32_t start = ruleIndex - (U_PARSE_CONTEXT_LEN - 1); |
| if(start < 0) { |
| start = 0; |
| } else if(start > 0 && U16_IS_TRAIL(rules->charAt(start))) { |
| ++start; |
| } |
| int32_t length = ruleIndex - start; |
| rules->extract(start, length, parseError->preContext); |
| parseError->preContext[length] = 0; |
| |
| // starting from ruleIndex |
| length = rules->length() - ruleIndex; |
| if(length >= U_PARSE_CONTEXT_LEN) { |
| length = U_PARSE_CONTEXT_LEN - 1; |
| if(U16_IS_LEAD(rules->charAt(ruleIndex + length - 1))) { |
| --length; |
| } |
| } |
| rules->extract(ruleIndex, length, parseError->postContext); |
| parseError->postContext[length] = 0; |
| } |
| |
| UBool |
| CollationRuleParser::isSyntaxChar(UChar32 c) { |
| return 0x21 <= c && c <= 0x7e && |
| (c <= 0x2f || (0x3a <= c && c <= 0x40) || |
| (0x5b <= c && c <= 0x60) || (0x7b <= c)); |
| } |
| |
| int32_t |
| CollationRuleParser::skipWhiteSpace(int32_t i) const { |
| while(i < rules->length() && PatternProps::isWhiteSpace(rules->charAt(i))) { |
| ++i; |
| } |
| return i; |
| } |
| |
| U_NAMESPACE_END |
| |
| #endif // !UCONFIG_NO_COLLATION |
