| /* |
| ****************************************************************************** |
| * |
| * Copyright (C) 2008-2009, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| * |
| ****************************************************************************** |
| * file name: uspoof_wsconf.cpp |
| * encoding: US-ASCII |
| * tab size: 8 (not used) |
| * indentation:4 |
| * |
| * created on: 2009Jan05 (refactoring earlier files) |
| * created by: Andy Heninger |
| * |
| * Internal functions for compililing Whole Script confusable source data |
| * into its binary (runtime) form. The binary data format is described |
| * in uspoof_impl.h |
| */ |
| |
| #include "unicode/utypes.h" |
| #include "unicode/uspoof.h" |
| |
| #if !UCONFIG_NO_NORMALIZATION |
| |
| #if !UCONFIG_NO_REGULAR_EXPRESSIONS |
| |
| #include "unicode/unorm.h" |
| #include "unicode/uregex.h" |
| #include "unicode/ustring.h" |
| #include "cmemory.h" |
| #include "uspoof_impl.h" |
| #include "uhash.h" |
| #include "uvector.h" |
| #include "uassert.h" |
| #include "uspoof_wsconf.h" |
| |
| U_NAMESPACE_USE |
| |
| |
| // Regular expression for parsing a line from the Unicode file confusablesWholeScript.txt |
| // Example Lines: |
| // 006F ; Latn; Deva; A # (o) LATIN SMALL LETTER O |
| // 0048..0049 ; Latn; Grek; A # [2] (H..I) LATIN CAPITAL LETTER H..LATIN CAPITAL LETTER I |
| // | | | | |
| // | | | |---- Which table, Any Case or Lower Case (A or L) |
| // | | |----------Target script. We need this. |
| // | |----------------Src script. Should match the script of the source |
| // | code points. Beyond checking that, we don't keep it. |
| // |--------------------------------Source code points or range. |
| // |
| // The expression will match _all_ lines, including erroneous lines. |
| // The result of the parse is returned via the contents of the (match) groups. |
| static const char *parseExp = |
| |
| "(?m)" // Multi-line mode |
| "^([ \\t]*(?:#.*?)?)$" // A blank or comment line. Matches Group 1. |
| "|^(?:" // OR |
| "\\s*([0-9A-F]{4,})(?:..([0-9A-F]{4,}))?\\s*;" // Code point range. Groups 2 and 3. |
| "\\s*([A-Za-z]+)\\s*;" // The source script. Group 4. |
| "\\s*([A-Za-z]+)\\s*;" // The target script. Group 5. |
| "\\s*(?:(A)|(L))" // The table A or L. Group 6 or 7 |
| "[ \\t]*(?:#.*?)?" // Trailing commment |
| ")$|" // OR |
| "^(.*?)$"; // An error line. Group 8. |
| // Any line not matching the preceding |
| // parts of the expression.will match |
| // this, and thus be flagged as an error |
| |
| |
| // Extract a regular expression match group into a char * string. |
| // The group must contain only invariant characters. |
| // Used for script names |
| // |
| static void extractGroup( |
| URegularExpression *e, int32_t group, char *destBuf, int32_t destCapacity, UErrorCode &status) { |
| |
| UChar ubuf[50]; |
| ubuf[0] = 0; |
| destBuf[0] = 0; |
| int32_t len = uregex_group(e, group, ubuf, 50, &status); |
| if (U_FAILURE(status) || len == -1 || len >= destCapacity) { |
| return; |
| } |
| UnicodeString s(FALSE, ubuf, len); // Aliasing constructor |
| s.extract(0, len, destBuf, destCapacity, US_INV); |
| } |
| |
| |
| |
| // Build the Whole Script Confusable data |
| // |
| // TODO: Reorganize. Either get rid of the WSConfusableDataBuilder class, |
| // because everything is local to this one build function anyhow, |
| // OR |
| // break this function into more reasonably sized pieces, with |
| // state in WSConfusableDataBuilder. |
| // |
| void buildWSConfusableData(SpoofImpl *spImpl, const char * confusablesWS, |
| int32_t confusablesWSLen, UParseError *pe, UErrorCode &status) |
| { |
| if (U_FAILURE(status)) { |
| return; |
| } |
| URegularExpression *parseRegexp = NULL; |
| int32_t inputLen = 0; |
| UChar *input = NULL; |
| int32_t lineNum = 0; |
| |
| UVector *scriptSets = NULL; |
| uint32_t rtScriptSetsCount = 2; |
| |
| UTrie2 *anyCaseTrie = NULL; |
| UTrie2 *lowerCaseTrie = NULL; |
| |
| anyCaseTrie = utrie2_open(0, 0, &status); |
| lowerCaseTrie = utrie2_open(0, 0, &status); |
| |
| |
| // The scriptSets vector provides a mapping from TRIE values to the set of scripts. |
| // |
| // Reserved TRIE values: |
| // 0: Code point has no whole script confusables. |
| // 1: Code point is of script Common or Inherited. |
| // These code points do not participate in whole script confusable detection. |
| // (This is logically equivalent to saying that they contain confusables in |
| // all scripts) |
| // |
| // Because Trie values are indexes into the ScriptSets vector, pre-fill |
| // vector positions 0 and 1 to avoid conflicts with the reserved values. |
| |
| scriptSets = new UVector(status); |
| if (scriptSets == NULL) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| goto cleanup; |
| } |
| scriptSets->addElement((void *)NULL, status); |
| scriptSets->addElement((void *)NULL, status); |
| |
| // Convert the user input data from UTF-8 to UChar (UTF-16) |
| u_strFromUTF8(NULL, 0, &inputLen, confusablesWS, confusablesWSLen, &status); |
| if (status != U_BUFFER_OVERFLOW_ERROR) { |
| goto cleanup; |
| } |
| status = U_ZERO_ERROR; |
| input = static_cast<UChar *>(uprv_malloc((inputLen+1) * sizeof(UChar))); |
| if (input == NULL) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| goto cleanup; |
| } |
| u_strFromUTF8(input, inputLen+1, NULL, confusablesWS, confusablesWSLen, &status); |
| |
| |
| |
| parseRegexp = uregex_openC(parseExp, 0, NULL, &status); |
| |
| // Zap any Byte Order Mark at the start of input. Changing it to a space is benign |
| // given the syntax of the input. |
| if (*input == 0xfeff) { |
| *input = 0x20; |
| } |
| |
| // Parse the input, one line per iteration of this loop. |
| uregex_setText(parseRegexp, input, inputLen, &status); |
| while (uregex_findNext(parseRegexp, &status)) { |
| lineNum++; |
| UChar line[200]; |
| uregex_group(parseRegexp, 0, line, 200, &status); |
| if (uregex_start(parseRegexp, 1, &status) >= 0) { |
| // this was a blank or comment line. |
| continue; |
| } |
| if (uregex_start(parseRegexp, 8, &status) >= 0) { |
| // input file syntax error. |
| status = U_PARSE_ERROR; |
| goto cleanup; |
| } |
| if (U_FAILURE(status)) { |
| goto cleanup; |
| } |
| |
| // Pick up the start and optional range end code points from the parsed line. |
| UChar32 startCodePoint = SpoofImpl::ScanHex( |
| input, uregex_start(parseRegexp, 2, &status), uregex_end(parseRegexp, 2, &status), status); |
| UChar32 endCodePoint = startCodePoint; |
| if (uregex_start(parseRegexp, 3, &status) >=0) { |
| endCodePoint = SpoofImpl::ScanHex( |
| input, uregex_start(parseRegexp, 3, &status), uregex_end(parseRegexp, 3, &status), status); |
| } |
| |
| // Extract the two script names from the source line. We need these in an 8 bit |
| // default encoding (will be EBCDIC on IBM mainframes) in order to pass them on |
| // to the ICU u_getPropertyValueEnum() function. Ugh. |
| char srcScriptName[20]; |
| char targScriptName[20]; |
| extractGroup(parseRegexp, 4, srcScriptName, sizeof(srcScriptName), status); |
| extractGroup(parseRegexp, 5, targScriptName, sizeof(targScriptName), status); |
| UScriptCode srcScript = |
| static_cast<UScriptCode>(u_getPropertyValueEnum(UCHAR_SCRIPT, srcScriptName)); |
| UScriptCode targScript = |
| static_cast<UScriptCode>(u_getPropertyValueEnum(UCHAR_SCRIPT, targScriptName)); |
| if (U_FAILURE(status)) { |
| goto cleanup; |
| } |
| if (srcScript == USCRIPT_INVALID_CODE || targScript == USCRIPT_INVALID_CODE) { |
| status = U_INVALID_FORMAT_ERROR; |
| goto cleanup; |
| } |
| |
| // select the table - (A) any case or (L) lower case only |
| UTrie2 *table = anyCaseTrie; |
| if (uregex_start(parseRegexp, 7, &status) >= 0) { |
| table = lowerCaseTrie; |
| } |
| |
| // Build the set of scripts containing confusable characters for |
| // the code point(s) specified in this input line. |
| // Sanity check that the script of the source code point is the same |
| // as the source script indicated in the input file. Failure of this check is |
| // an error in the input file. |
| // Include the source script in the set (needed for Mixed Script Confusable detection). |
| // |
| UChar32 cp; |
| for (cp=startCodePoint; cp<=endCodePoint; cp++) { |
| int32_t setIndex = utrie2_get32(table, cp); |
| BuilderScriptSet *bsset = NULL; |
| if (setIndex > 0) { |
| U_ASSERT(setIndex < scriptSets->size()); |
| bsset = static_cast<BuilderScriptSet *>(scriptSets->elementAt(setIndex)); |
| } else { |
| bsset = new BuilderScriptSet(); |
| if (bsset == NULL) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| goto cleanup; |
| } |
| bsset->codePoint = cp; |
| bsset->trie = table; |
| bsset->sset = new ScriptSet(); |
| setIndex = scriptSets->size(); |
| bsset->index = setIndex; |
| bsset->rindex = 0; |
| if (bsset->sset == NULL) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| goto cleanup; |
| } |
| scriptSets->addElement(bsset, status); |
| utrie2_set32(table, cp, setIndex, &status); |
| } |
| bsset->sset->Union(targScript); |
| bsset->sset->Union(srcScript); |
| |
| if (U_FAILURE(status)) { |
| goto cleanup; |
| } |
| UScriptCode cpScript = uscript_getScript(cp, &status); |
| if (cpScript != srcScript) { |
| status = U_INVALID_FORMAT_ERROR; |
| goto cleanup; |
| } |
| } |
| } |
| |
| // Eliminate duplicate script sets. At this point we have a separate |
| // script set for every code point that had data in the input file. |
| // |
| // We eliminate underlying ScriptSet objects, not the BuildScriptSets that wrap them |
| // |
| // printf("Number of scriptSets: %d\n", scriptSets->size()); |
| { |
| int32_t duplicateCount = 0; |
| rtScriptSetsCount = 2; |
| for (int32_t outeri=2; outeri<scriptSets->size(); outeri++) { |
| BuilderScriptSet *outerSet = static_cast<BuilderScriptSet *>(scriptSets->elementAt(outeri)); |
| if (outerSet->index != static_cast<uint32_t>(outeri)) { |
| // This set was already identified as a duplicate. |
| // It will not be allocated a position in the runtime array of ScriptSets. |
| continue; |
| } |
| outerSet->rindex = rtScriptSetsCount++; |
| for (int32_t inneri=outeri+1; inneri<scriptSets->size(); inneri++) { |
| BuilderScriptSet *innerSet = static_cast<BuilderScriptSet *>(scriptSets->elementAt(inneri)); |
| if (*(outerSet->sset) == *(innerSet->sset) && outerSet->sset != innerSet->sset) { |
| delete innerSet->sset; |
| innerSet->scriptSetOwned = FALSE; |
| innerSet->sset = outerSet->sset; |
| innerSet->index = outeri; |
| innerSet->rindex = outerSet->rindex; |
| duplicateCount++; |
| } |
| // But this doesn't get all. We need to fix the TRIE. |
| } |
| } |
| // printf("Number of distinct script sets: %d\n", rtScriptSetsCount); |
| } |
| |
| |
| |
| // Update the Trie values to be reflect the run time script indexes (after duplicate merging). |
| // (Trie Values 0 and 1 are reserved, and the corresponding slots in scriptSets |
| // are unused, which is why the loop index starts at 2.) |
| { |
| for (int32_t i=2; i<scriptSets->size(); i++) { |
| BuilderScriptSet *bSet = static_cast<BuilderScriptSet *>(scriptSets->elementAt(i)); |
| if (bSet->rindex != (uint32_t)i) { |
| utrie2_set32(bSet->trie, bSet->codePoint, bSet->rindex, &status); |
| } |
| } |
| } |
| |
| // For code points with script==Common or script==Inherited, |
| // Set the reserved value of 1 into both Tries. These characters do not participate |
| // in Whole Script Confusable detection; this reserved value is the means |
| // by which they are detected. |
| { |
| UnicodeSet ignoreSet; |
| ignoreSet.applyIntPropertyValue(UCHAR_SCRIPT, USCRIPT_COMMON, status); |
| UnicodeSet inheritedSet; |
| inheritedSet.applyIntPropertyValue(UCHAR_SCRIPT, USCRIPT_INHERITED, status); |
| ignoreSet.addAll(inheritedSet); |
| for (int32_t rn=0; rn<ignoreSet.getRangeCount(); rn++) { |
| UChar32 rangeStart = ignoreSet.getRangeStart(rn); |
| UChar32 rangeEnd = ignoreSet.getRangeEnd(rn); |
| utrie2_setRange32(anyCaseTrie, rangeStart, rangeEnd, 1, TRUE, &status); |
| utrie2_setRange32(lowerCaseTrie, rangeStart, rangeEnd, 1, TRUE, &status); |
| } |
| } |
| |
| // Serialize the data to the Spoof Detector |
| { |
| utrie2_freeze(anyCaseTrie, UTRIE2_16_VALUE_BITS, &status); |
| int32_t size = utrie2_serialize(anyCaseTrie, NULL, 0, &status); |
| // printf("Any case Trie size: %d\n", size); |
| if (status != U_BUFFER_OVERFLOW_ERROR) { |
| goto cleanup; |
| } |
| status = U_ZERO_ERROR; |
| spImpl->fSpoofData->fRawData->fAnyCaseTrie = spImpl->fSpoofData->fMemLimit; |
| spImpl->fSpoofData->fRawData->fAnyCaseTrieLength = size; |
| spImpl->fSpoofData->fAnyCaseTrie = anyCaseTrie; |
| void *where = spImpl->fSpoofData->reserveSpace(size, status); |
| utrie2_serialize(anyCaseTrie, where, size, &status); |
| |
| utrie2_freeze(lowerCaseTrie, UTRIE2_16_VALUE_BITS, &status); |
| size = utrie2_serialize(lowerCaseTrie, NULL, 0, &status); |
| // printf("Lower case Trie size: %d\n", size); |
| if (status != U_BUFFER_OVERFLOW_ERROR) { |
| goto cleanup; |
| } |
| status = U_ZERO_ERROR; |
| spImpl->fSpoofData->fRawData->fLowerCaseTrie = spImpl->fSpoofData->fMemLimit; |
| spImpl->fSpoofData->fRawData->fLowerCaseTrieLength = size; |
| spImpl->fSpoofData->fLowerCaseTrie = lowerCaseTrie; |
| where = spImpl->fSpoofData->reserveSpace(size, status); |
| utrie2_serialize(lowerCaseTrie, where, size, &status); |
| |
| spImpl->fSpoofData->fRawData->fScriptSets = spImpl->fSpoofData->fMemLimit; |
| spImpl->fSpoofData->fRawData->fScriptSetsLength = rtScriptSetsCount; |
| ScriptSet *rtScriptSets = static_cast<ScriptSet *> |
| (spImpl->fSpoofData->reserveSpace(rtScriptSetsCount * sizeof(ScriptSet), status)); |
| uint32_t rindex = 2; |
| for (int32_t i=2; i<scriptSets->size(); i++) { |
| BuilderScriptSet *bSet = static_cast<BuilderScriptSet *>(scriptSets->elementAt(i)); |
| if (bSet->rindex < rindex) { |
| // We have already copied this script set to the serialized data. |
| continue; |
| } |
| U_ASSERT(rindex == bSet->rindex); |
| rtScriptSets[rindex] = *bSet->sset; // Assignment of a ScriptSet just copies the bits. |
| rindex++; |
| } |
| } |
| |
| // Open new utrie2s from the serialized data. We don't want to keep the ones |
| // we just built because we would then have two copies of the data, one internal to |
| // the utries that we have already constructed, and one in the serialized data area. |
| // An alternative would be to not pre-serialize the Trie data, but that makes the |
| // spoof detector data different, depending on how the detector was constructed. |
| // It's simpler to keep the data always the same. |
| |
| spImpl->fSpoofData->fAnyCaseTrie = utrie2_openFromSerialized( |
| UTRIE2_16_VALUE_BITS, |
| (const char *)spImpl->fSpoofData->fRawData + spImpl->fSpoofData->fRawData->fAnyCaseTrie, |
| spImpl->fSpoofData->fRawData->fAnyCaseTrieLength, |
| NULL, |
| &status); |
| |
| spImpl->fSpoofData->fLowerCaseTrie = utrie2_openFromSerialized( |
| UTRIE2_16_VALUE_BITS, |
| (const char *)spImpl->fSpoofData->fRawData + spImpl->fSpoofData->fRawData->fLowerCaseTrie, |
| spImpl->fSpoofData->fRawData->fAnyCaseTrieLength, |
| NULL, |
| &status); |
| |
| |
| |
| cleanup: |
| if (U_FAILURE(status)) { |
| pe->line = lineNum; |
| } |
| uregex_close(parseRegexp); |
| uprv_free(input); |
| |
| int32_t i; |
| for (i=0; i<scriptSets->size(); i++) { |
| BuilderScriptSet *bsset = static_cast<BuilderScriptSet *>(scriptSets->elementAt(i)); |
| delete bsset; |
| } |
| delete scriptSets; |
| utrie2_close(anyCaseTrie); |
| utrie2_close(lowerCaseTrie); |
| return; |
| } |
| |
| |
| |
| |
| |
| BuilderScriptSet::BuilderScriptSet() { |
| codePoint = -1; |
| trie = NULL; |
| sset = NULL; |
| index = 0; |
| rindex = 0; |
| scriptSetOwned = TRUE; |
| } |
| |
| BuilderScriptSet::~BuilderScriptSet() { |
| if (scriptSetOwned) { |
| delete sset; |
| } |
| } |
| |
| #endif |
| #endif // !UCONFIG_NO_REGULAR_EXPRESSIONS |
| |
