| /* | |
| ********************************************************************** | |
| * Copyright (C) 1999-2011, International Business Machines | |
| * Corporation and others. All Rights Reserved. | |
| ********************************************************************** | |
| * Date Name Description | |
| * 11/17/99 aliu Creation. | |
| ********************************************************************** | |
| */ | |
| #include "unicode/utypes.h" | |
| #if !UCONFIG_NO_TRANSLITERATION | |
| #include "starboard/client_porting/poem/string_poem.h" | |
| #include "unicode/unifilt.h" | |
| #include "unicode/uniset.h" | |
| #include "cpdtrans.h" | |
| #include "uvector.h" | |
| #include "tridpars.h" | |
| #include "cmemory.h" | |
| // keep in sync with Transliterator | |
| //static const UChar ID_SEP = 0x002D; /*-*/ | |
| static const UChar ID_DELIM = 0x003B; /*;*/ | |
| static const UChar NEWLINE = 10; | |
| static const UChar COLON_COLON[] = {0x3A, 0x3A, 0}; //"::" | |
| U_NAMESPACE_BEGIN | |
| const UChar CompoundTransliterator::PASS_STRING[] = { 0x0025, 0x0050, 0x0061, 0x0073, 0x0073, 0 }; // "%Pass" | |
| UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CompoundTransliterator) | |
| /** | |
| * Constructs a new compound transliterator given an array of | |
| * transliterators. The array of transliterators may be of any | |
| * length, including zero or one, however, useful compound | |
| * transliterators have at least two components. | |
| * @param transliterators array of <code>Transliterator</code> | |
| * objects | |
| * @param transliteratorCount The number of | |
| * <code>Transliterator</code> objects in transliterators. | |
| * @param filter the filter. Any character for which | |
| * <tt>filter.contains()</tt> returns <tt>false</tt> will not be | |
| * altered by this transliterator. If <tt>filter</tt> is | |
| * <tt>null</tt> then no filtering is applied. | |
| */ | |
| CompoundTransliterator::CompoundTransliterator( | |
| Transliterator* const transliterators[], | |
| int32_t transliteratorCount, | |
| UnicodeFilter* adoptedFilter) : | |
| Transliterator(joinIDs(transliterators, transliteratorCount), adoptedFilter), | |
| trans(0), count(0), numAnonymousRBTs(0) { | |
| setTransliterators(transliterators, transliteratorCount); | |
| } | |
| /** | |
| * Splits an ID of the form "ID;ID;..." into a compound using each | |
| * of the IDs. | |
| * @param id of above form | |
| * @param forward if false, does the list in reverse order, and | |
| * takes the inverse of each ID. | |
| */ | |
| CompoundTransliterator::CompoundTransliterator(const UnicodeString& id, | |
| UTransDirection direction, | |
| UnicodeFilter* adoptedFilter, | |
| UParseError& /*parseError*/, | |
| UErrorCode& status) : | |
| Transliterator(id, adoptedFilter), | |
| trans(0), numAnonymousRBTs(0) { | |
| // TODO add code for parseError...currently unused, but | |
| // later may be used by parsing code... | |
| init(id, direction, TRUE, status); | |
| } | |
| CompoundTransliterator::CompoundTransliterator(const UnicodeString& id, | |
| UParseError& /*parseError*/, | |
| UErrorCode& status) : | |
| Transliterator(id, 0), // set filter to 0 here! | |
| trans(0), numAnonymousRBTs(0) { | |
| // TODO add code for parseError...currently unused, but | |
| // later may be used by parsing code... | |
| init(id, UTRANS_FORWARD, TRUE, status); | |
| } | |
| /** | |
| * Private constructor for use of TransliteratorAlias | |
| */ | |
| CompoundTransliterator::CompoundTransliterator(const UnicodeString& newID, | |
| UVector& list, | |
| UnicodeFilter* adoptedFilter, | |
| int32_t anonymousRBTs, | |
| UParseError& /*parseError*/, | |
| UErrorCode& status) : | |
| Transliterator(newID, adoptedFilter), | |
| trans(0), numAnonymousRBTs(anonymousRBTs) | |
| { | |
| init(list, UTRANS_FORWARD, FALSE, status); | |
| } | |
| /** | |
| * Private constructor for Transliterator from a vector of | |
| * transliterators. The caller is responsible for fixing up the | |
| * ID. | |
| */ | |
| CompoundTransliterator::CompoundTransliterator(UVector& list, | |
| UParseError& /*parseError*/, | |
| UErrorCode& status) : | |
| Transliterator(UnicodeString(), NULL), | |
| trans(0), numAnonymousRBTs(0) | |
| { | |
| // TODO add code for parseError...currently unused, but | |
| // later may be used by parsing code... | |
| init(list, UTRANS_FORWARD, FALSE, status); | |
| // assume caller will fixup ID | |
| } | |
| CompoundTransliterator::CompoundTransliterator(UVector& list, | |
| int32_t anonymousRBTs, | |
| UParseError& /*parseError*/, | |
| UErrorCode& status) : | |
| Transliterator(UnicodeString(), NULL), | |
| trans(0), numAnonymousRBTs(anonymousRBTs) | |
| { | |
| init(list, UTRANS_FORWARD, FALSE, status); | |
| } | |
| /** | |
| * Finish constructing a transliterator: only to be called by | |
| * constructors. Before calling init(), set trans and filter to NULL. | |
| * @param id the id containing ';'-separated entries | |
| * @param direction either FORWARD or REVERSE | |
| * @param idSplitPoint the index into id at which the | |
| * adoptedSplitTransliterator should be inserted, if there is one, or | |
| * -1 if there is none. | |
| * @param adoptedSplitTransliterator a transliterator to be inserted | |
| * before the entry at offset idSplitPoint in the id string. May be | |
| * NULL to insert no entry. | |
| * @param fixReverseID if TRUE, then reconstruct the ID of reverse | |
| * entries by calling getID() of component entries. Some constructors | |
| * do not require this because they apply a facade ID anyway. | |
| * @param status the error code indicating success or failure | |
| */ | |
| void CompoundTransliterator::init(const UnicodeString& id, | |
| UTransDirection direction, | |
| UBool fixReverseID, | |
| UErrorCode& status) { | |
| // assert(trans == 0); | |
| if (U_FAILURE(status)) { | |
| return; | |
| } | |
| UVector list(status); | |
| UnicodeSet* compoundFilter = NULL; | |
| UnicodeString regenID; | |
| if (!TransliteratorIDParser::parseCompoundID(id, direction, | |
| regenID, list, compoundFilter)) { | |
| status = U_INVALID_ID; | |
| delete compoundFilter; | |
| return; | |
| } | |
| TransliteratorIDParser::instantiateList(list, status); | |
| init(list, direction, fixReverseID, status); | |
| if (compoundFilter != NULL) { | |
| adoptFilter(compoundFilter); | |
| } | |
| } | |
| /** | |
| * Finish constructing a transliterator: only to be called by | |
| * constructors. Before calling init(), set trans and filter to NULL. | |
| * @param list a vector of transliterator objects to be adopted. It | |
| * should NOT be empty. The list should be in declared order. That | |
| * is, it should be in the FORWARD order; if direction is REVERSE then | |
| * the list order will be reversed. | |
| * @param direction either FORWARD or REVERSE | |
| * @param fixReverseID if TRUE, then reconstruct the ID of reverse | |
| * entries by calling getID() of component entries. Some constructors | |
| * do not require this because they apply a facade ID anyway. | |
| * @param status the error code indicating success or failure | |
| */ | |
| void CompoundTransliterator::init(UVector& list, | |
| UTransDirection direction, | |
| UBool fixReverseID, | |
| UErrorCode& status) { | |
| // assert(trans == 0); | |
| // Allocate array | |
| if (U_SUCCESS(status)) { | |
| count = list.size(); | |
| trans = (Transliterator **)uprv_malloc(count * sizeof(Transliterator *)); | |
| /* test for NULL */ | |
| if (trans == 0) { | |
| status = U_MEMORY_ALLOCATION_ERROR; | |
| return; | |
| } | |
| } | |
| if (U_FAILURE(status) || trans == 0) { | |
| // assert(trans == 0); | |
| return; | |
| } | |
| // Move the transliterators from the vector into an array. | |
| // Reverse the order if necessary. | |
| int32_t i; | |
| for (i=0; i<count; ++i) { | |
| int32_t j = (direction == UTRANS_FORWARD) ? i : count - 1 - i; | |
| trans[i] = (Transliterator*) list.elementAt(j); | |
| } | |
| // If the direction is UTRANS_REVERSE then we may need to fix the | |
| // ID. | |
| if (direction == UTRANS_REVERSE && fixReverseID) { | |
| UnicodeString newID; | |
| for (i=0; i<count; ++i) { | |
| if (i > 0) { | |
| newID.append(ID_DELIM); | |
| } | |
| newID.append(trans[i]->getID()); | |
| } | |
| setID(newID); | |
| } | |
| computeMaximumContextLength(); | |
| } | |
| /** | |
| * Return the IDs of the given list of transliterators, concatenated | |
| * with ID_DELIM delimiting them. Equivalent to the perlish expression | |
| * join(ID_DELIM, map($_.getID(), transliterators). | |
| */ | |
| UnicodeString CompoundTransliterator::joinIDs(Transliterator* const transliterators[], | |
| int32_t transCount) { | |
| UnicodeString id; | |
| for (int32_t i=0; i<transCount; ++i) { | |
| if (i > 0) { | |
| id.append(ID_DELIM); | |
| } | |
| id.append(transliterators[i]->getID()); | |
| } | |
| return id; // Return temporary | |
| } | |
| /** | |
| * Copy constructor. | |
| */ | |
| CompoundTransliterator::CompoundTransliterator(const CompoundTransliterator& t) : | |
| Transliterator(t), trans(0), count(0), numAnonymousRBTs(-1) { | |
| *this = t; | |
| } | |
| /** | |
| * Destructor | |
| */ | |
| CompoundTransliterator::~CompoundTransliterator() { | |
| freeTransliterators(); | |
| } | |
| void CompoundTransliterator::freeTransliterators(void) { | |
| if (trans != 0) { | |
| for (int32_t i=0; i<count; ++i) { | |
| delete trans[i]; | |
| } | |
| uprv_free(trans); | |
| } | |
| trans = 0; | |
| count = 0; | |
| } | |
| /** | |
| * Assignment operator. | |
| */ | |
| CompoundTransliterator& CompoundTransliterator::operator=( | |
| const CompoundTransliterator& t) | |
| { | |
| Transliterator::operator=(t); | |
| int32_t i = 0; | |
| UBool failed = FALSE; | |
| if (trans != NULL) { | |
| for (i=0; i<count; ++i) { | |
| delete trans[i]; | |
| trans[i] = 0; | |
| } | |
| } | |
| if (t.count > count) { | |
| if (trans != NULL) { | |
| uprv_free(trans); | |
| } | |
| trans = (Transliterator **)uprv_malloc(t.count * sizeof(Transliterator *)); | |
| } | |
| count = t.count; | |
| if (trans != NULL) { | |
| for (i=0; i<count; ++i) { | |
| trans[i] = t.trans[i]->clone(); | |
| if (trans[i] == NULL) { | |
| failed = TRUE; | |
| break; | |
| } | |
| } | |
| } | |
| // if memory allocation failed delete backwards trans array | |
| if (failed && i > 0) { | |
| int32_t n; | |
| for (n = i-1; n >= 0; n--) { | |
| uprv_free(trans[n]); | |
| trans[n] = NULL; | |
| } | |
| } | |
| numAnonymousRBTs = t.numAnonymousRBTs; | |
| return *this; | |
| } | |
| /** | |
| * Transliterator API. | |
| */ | |
| Transliterator* CompoundTransliterator::clone(void) const { | |
| return new CompoundTransliterator(*this); | |
| } | |
| /** | |
| * Returns the number of transliterators in this chain. | |
| * @return number of transliterators in this chain. | |
| */ | |
| int32_t CompoundTransliterator::getCount(void) const { | |
| return count; | |
| } | |
| /** | |
| * Returns the transliterator at the given index in this chain. | |
| * @param index index into chain, from 0 to <code>getCount() - 1</code> | |
| * @return transliterator at the given index | |
| */ | |
| const Transliterator& CompoundTransliterator::getTransliterator(int32_t index) const { | |
| return *trans[index]; | |
| } | |
| void CompoundTransliterator::setTransliterators(Transliterator* const transliterators[], | |
| int32_t transCount) { | |
| Transliterator** a = (Transliterator **)uprv_malloc(transCount * sizeof(Transliterator *)); | |
| if (a == NULL) { | |
| return; | |
| } | |
| int32_t i = 0; | |
| UBool failed = FALSE; | |
| for (i=0; i<transCount; ++i) { | |
| a[i] = transliterators[i]->clone(); | |
| if (a[i] == NULL) { | |
| failed = TRUE; | |
| break; | |
| } | |
| } | |
| if (failed && i > 0) { | |
| int32_t n; | |
| for (n = i-1; n >= 0; n--) { | |
| uprv_free(a[n]); | |
| a[n] = NULL; | |
| } | |
| return; | |
| } | |
| adoptTransliterators(a, transCount); | |
| } | |
| void CompoundTransliterator::adoptTransliterators(Transliterator* adoptedTransliterators[], | |
| int32_t transCount) { | |
| // First free trans[] and set count to zero. Once this is done, | |
| // orphan the filter. Set up the new trans[]. | |
| freeTransliterators(); | |
| trans = adoptedTransliterators; | |
| count = transCount; | |
| computeMaximumContextLength(); | |
| setID(joinIDs(trans, count)); | |
| } | |
| /** | |
| * Append c to buf, unless buf is empty or buf already ends in c. | |
| */ | |
| static void _smartAppend(UnicodeString& buf, UChar c) { | |
| if (buf.length() != 0 && | |
| buf.charAt(buf.length() - 1) != c) { | |
| buf.append(c); | |
| } | |
| } | |
| UnicodeString& CompoundTransliterator::toRules(UnicodeString& rulesSource, | |
| UBool escapeUnprintable) const { | |
| // We do NOT call toRules() on our component transliterators, in | |
| // general. If we have several rule-based transliterators, this | |
| // yields a concatenation of the rules -- not what we want. We do | |
| // handle compound RBT transliterators specially -- those for which | |
| // compoundRBTIndex >= 0. For the transliterator at compoundRBTIndex, | |
| // we do call toRules() recursively. | |
| rulesSource.truncate(0); | |
| if (numAnonymousRBTs >= 1 && getFilter() != NULL) { | |
| // If we are a compound RBT and if we have a global | |
| // filter, then emit it at the top. | |
| UnicodeString pat; | |
| rulesSource.append(COLON_COLON, 2).append(getFilter()->toPattern(pat, escapeUnprintable)).append(ID_DELIM); | |
| } | |
| for (int32_t i=0; i<count; ++i) { | |
| UnicodeString rule; | |
| // Anonymous RuleBasedTransliterators (inline rules and | |
| // ::BEGIN/::END blocks) are given IDs that begin with | |
| // "%Pass": use toRules() to write all the rules to the output | |
| // (and insert "::Null;" if we have two in a row) | |
| if (trans[i]->getID().startsWith(PASS_STRING, 5)) { | |
| trans[i]->toRules(rule, escapeUnprintable); | |
| if (numAnonymousRBTs > 1 && i > 0 && trans[i - 1]->getID().startsWith(PASS_STRING, 5)) | |
| rule = UNICODE_STRING_SIMPLE("::Null;") + rule; | |
| // we also use toRules() on CompoundTransliterators (which we | |
| // check for by looking for a semicolon in the ID)-- this gets | |
| // the list of their child transliterators output in the right | |
| // format | |
| } else if (trans[i]->getID().indexOf(ID_DELIM) >= 0) { | |
| trans[i]->toRules(rule, escapeUnprintable); | |
| // for everything else, use Transliterator::toRules() | |
| } else { | |
| trans[i]->Transliterator::toRules(rule, escapeUnprintable); | |
| } | |
| _smartAppend(rulesSource, NEWLINE); | |
| rulesSource.append(rule); | |
| _smartAppend(rulesSource, ID_DELIM); | |
| } | |
| return rulesSource; | |
| } | |
| /** | |
| * Implement Transliterator framework | |
| */ | |
| void CompoundTransliterator::handleGetSourceSet(UnicodeSet& result) const { | |
| UnicodeSet set; | |
| result.clear(); | |
| for (int32_t i=0; i<count; ++i) { | |
| result.addAll(trans[i]->getSourceSet(set)); | |
| // Take the example of Hiragana-Latin. This is really | |
| // Hiragana-Katakana; Katakana-Latin. The source set of | |
| // these two is roughly [:Hiragana:] and [:Katakana:]. | |
| // But the source set for the entire transliterator is | |
| // actually [:Hiragana:] ONLY -- that is, the first | |
| // non-empty source set. | |
| // This is a heuristic, and not 100% reliable. | |
| if (!result.isEmpty()) { | |
| break; | |
| } | |
| } | |
| } | |
| /** | |
| * Override Transliterator framework | |
| */ | |
| UnicodeSet& CompoundTransliterator::getTargetSet(UnicodeSet& result) const { | |
| UnicodeSet set; | |
| result.clear(); | |
| for (int32_t i=0; i<count; ++i) { | |
| // This is a heuristic, and not 100% reliable. | |
| result.addAll(trans[i]->getTargetSet(set)); | |
| } | |
| return result; | |
| } | |
| /** | |
| * Implements {@link Transliterator#handleTransliterate}. | |
| */ | |
| void CompoundTransliterator::handleTransliterate(Replaceable& text, UTransPosition& index, | |
| UBool incremental) const { | |
| /* Call each transliterator with the same contextStart and | |
| * start, but with the limit as modified | |
| * by preceding transliterators. The start index must be | |
| * reset for each transliterator to give each a chance to | |
| * transliterate the text. The initial contextStart index is known | |
| * to still point to the same place after each transliterator | |
| * is called because each transliterator will not change the | |
| * text between contextStart and the initial start index. | |
| * | |
| * IMPORTANT: After the first transliterator, each subsequent | |
| * transliterator only gets to transliterate text committed by | |
| * preceding transliterators; that is, the start (output | |
| * value) of transliterator i becomes the limit (input value) | |
| * of transliterator i+1. Finally, the overall limit is fixed | |
| * up before we return. | |
| * | |
| * Assumptions we make here: | |
| * (1) contextStart <= start <= limit <= contextLimit <= text.length() | |
| * (2) start <= start' <= limit' ;cursor doesn't move back | |
| * (3) start <= limit' ;text before cursor unchanged | |
| * - start' is the value of start after calling handleKT | |
| * - limit' is the value of limit after calling handleKT | |
| */ | |
| /** | |
| * Example: 3 transliterators. This example illustrates the | |
| * mechanics we need to implement. C, S, and L are the contextStart, | |
| * start, and limit. gl is the globalLimit. contextLimit is | |
| * equal to limit throughout. | |
| * | |
| * 1. h-u, changes hex to Unicode | |
| * | |
| * 4 7 a d 0 4 7 a | |
| * abc/u0061/u => abca/u | |
| * C S L C S L gl=f->a | |
| * | |
| * 2. upup, changes "x" to "XX" | |
| * | |
| * 4 7 a 4 7 a | |
| * abca/u => abcAA/u | |
| * C SL C S | |
| * L gl=a->b | |
| * 3. u-h, changes Unicode to hex | |
| * | |
| * 4 7 a 4 7 a d 0 3 | |
| * abcAA/u => abc/u0041/u0041/u | |
| * C S L C S | |
| * L gl=b->15 | |
| * 4. return | |
| * | |
| * 4 7 a d 0 3 | |
| * abc/u0041/u0041/u | |
| * C S L | |
| */ | |
| if (count < 1) { | |
| index.start = index.limit; | |
| return; // Short circuit for empty compound transliterators | |
| } | |
| // compoundLimit is the limit value for the entire compound | |
| // operation. We overwrite index.limit with the previous | |
| // index.start. After each transliteration, we update | |
| // compoundLimit for insertions or deletions that have happened. | |
| int32_t compoundLimit = index.limit; | |
| // compoundStart is the start for the entire compound | |
| // operation. | |
| int32_t compoundStart = index.start; | |
| int32_t delta = 0; // delta in length | |
| // Give each transliterator a crack at the run of characters. | |
| // See comments at the top of the method for more detail. | |
| for (int32_t i=0; i<count; ++i) { | |
| index.start = compoundStart; // Reset start | |
| int32_t limit = index.limit; | |
| if (index.start == index.limit) { | |
| // Short circuit for empty range | |
| break; | |
| } | |
| trans[i]->filteredTransliterate(text, index, incremental); | |
| // In a properly written transliterator, start == limit after | |
| // handleTransliterate() returns when incremental is false. | |
| // Catch cases where the subclass doesn't do this, and throw | |
| // an exception. (Just pinning start to limit is a bad idea, | |
| // because what's probably happening is that the subclass | |
| // isn't transliterating all the way to the end, and it should | |
| // in non-incremental mode.) | |
| if (!incremental && index.start != index.limit) { | |
| // We can't throw an exception, so just fudge things | |
| index.start = index.limit; | |
| } | |
| // Cumulative delta for insertions/deletions | |
| delta += index.limit - limit; | |
| if (incremental) { | |
| // In the incremental case, only allow subsequent | |
| // transliterators to modify what has already been | |
| // completely processed by prior transliterators. In the | |
| // non-incrmental case, allow each transliterator to | |
| // process the entire text. | |
| index.limit = index.start; | |
| } | |
| } | |
| compoundLimit += delta; | |
| // Start is good where it is -- where the last transliterator left | |
| // it. Limit needs to be put back where it was, modulo | |
| // adjustments for deletions/insertions. | |
| index.limit = compoundLimit; | |
| } | |
| /** | |
| * Sets the length of the longest context required by this transliterator. | |
| * This is <em>preceding</em> context. | |
| */ | |
| void CompoundTransliterator::computeMaximumContextLength(void) { | |
| int32_t max = 0; | |
| for (int32_t i=0; i<count; ++i) { | |
| int32_t len = trans[i]->getMaximumContextLength(); | |
| if (len > max) { | |
| max = len; | |
| } | |
| } | |
| setMaximumContextLength(max); | |
| } | |
| U_NAMESPACE_END | |
| #endif /* #if !UCONFIG_NO_TRANSLITERATION */ | |
| /* eof */ |