| // © 2016 and later: Unicode, Inc. and others. |
| // License & terms of use: http://www.unicode.org/copyright.html |
| /* |
| ******************************************************************************* |
| * |
| * Copyright (C) 2000-2015, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| * |
| ******************************************************************************* |
| * |
| * File reslist.cpp |
| * |
| * Modification History: |
| * |
| * Date Name Description |
| * 02/21/00 weiv Creation. |
| ******************************************************************************* |
| */ |
| |
| // Safer use of UnicodeString. |
| #ifndef UNISTR_FROM_CHAR_EXPLICIT |
| # define UNISTR_FROM_CHAR_EXPLICIT explicit |
| #endif |
| |
| // Less important, but still a good idea. |
| #ifndef UNISTR_FROM_STRING_EXPLICIT |
| # define UNISTR_FROM_STRING_EXPLICIT explicit |
| #endif |
| |
| #include <assert.h> |
| #include <iostream> |
| #include <set> |
| #include <stdio.h> |
| |
| #include "unicode/localpointer.h" |
| #include "reslist.h" |
| #include "unewdata.h" |
| #include "unicode/ures.h" |
| #include "unicode/putil.h" |
| #include "errmsg.h" |
| #include "filterrb.h" |
| #include "toolutil.h" |
| |
| #include "uarrsort.h" |
| #include "uelement.h" |
| #include "uhash.h" |
| #include "uinvchar.h" |
| #include "ustr_imp.h" |
| #include "unicode/utf16.h" |
| #include "uassert.h" |
| |
| /* |
| * Align binary data at a 16-byte offset from the start of the resource bundle, |
| * to be safe for any data type it may contain. |
| */ |
| #define BIN_ALIGNMENT 16 |
| |
| // This numeric constant must be at least 1. |
| // If StringResource.fNumUnitsSaved == 0 then the string occurs only once, |
| // and it makes no sense to move it to the pool bundle. |
| // The larger the threshold for fNumUnitsSaved |
| // the smaller the savings, and the smaller the pool bundle. |
| // We trade some total size reduction to reduce the pool bundle a bit, |
| // so that one can reasonably save data size by |
| // removing bundle files without rebuilding the pool bundle. |
| // This can also help to keep the pool and total (pool+local) string indexes |
| // within 16 bits, that is, within range of Table16 and Array16 containers. |
| #ifndef GENRB_MIN_16BIT_UNITS_SAVED_FOR_POOL_STRING |
| # define GENRB_MIN_16BIT_UNITS_SAVED_FOR_POOL_STRING 10 |
| #endif |
| |
| U_NAMESPACE_USE |
| |
| static UBool gIncludeCopyright = FALSE; |
| static UBool gUsePoolBundle = FALSE; |
| static UBool gIsDefaultFormatVersion = TRUE; |
| static int32_t gFormatVersion = 3; |
| |
| /* How do we store string values? */ |
| enum { |
| STRINGS_UTF16_V1, /* formatVersion 1: int length + UChars + NUL + padding to 4 bytes */ |
| STRINGS_UTF16_V2 /* formatVersion 2 & up: optional length in 1..3 UChars + UChars + NUL */ |
| }; |
| |
| static const int32_t MAX_IMPLICIT_STRING_LENGTH = 40; /* do not store the length explicitly for such strings */ |
| |
| static const ResFile kNoPoolBundle; |
| |
| /* |
| * res_none() returns the address of kNoResource, |
| * for use in non-error cases when no resource is to be added to the bundle. |
| * (NULL is used in error cases.) |
| */ |
| static SResource kNoResource; // TODO: const |
| |
| static UDataInfo dataInfo= { |
| sizeof(UDataInfo), |
| 0, |
| |
| U_IS_BIG_ENDIAN, |
| U_CHARSET_FAMILY, |
| sizeof(UChar), |
| 0, |
| |
| {0x52, 0x65, 0x73, 0x42}, /* dataFormat="ResB" */ |
| {1, 3, 0, 0}, /* formatVersion */ |
| {1, 4, 0, 0} /* dataVersion take a look at version inside parsed resb*/ |
| }; |
| |
| static const UVersionInfo gFormatVersions[4] = { /* indexed by a major-formatVersion integer */ |
| { 0, 0, 0, 0 }, |
| { 1, 3, 0, 0 }, |
| { 2, 0, 0, 0 }, |
| { 3, 0, 0, 0 } |
| }; |
| // Remember to update genrb.h GENRB_VERSION when changing the data format. |
| // (Or maybe we should remove GENRB_VERSION and report the ICU version number?) |
| |
| static uint8_t calcPadding(uint32_t size) { |
| /* returns space we need to pad */ |
| return (uint8_t) ((size % sizeof(uint32_t)) ? (sizeof(uint32_t) - (size % sizeof(uint32_t))) : 0); |
| |
| } |
| |
| void setIncludeCopyright(UBool val){ |
| gIncludeCopyright=val; |
| } |
| |
| UBool getIncludeCopyright(void){ |
| return gIncludeCopyright; |
| } |
| |
| void setFormatVersion(int32_t formatVersion) { |
| gIsDefaultFormatVersion = FALSE; |
| gFormatVersion = formatVersion; |
| } |
| |
| int32_t getFormatVersion() { |
| return gFormatVersion; |
| } |
| |
| void setUsePoolBundle(UBool use) { |
| gUsePoolBundle = use; |
| } |
| |
| // TODO: return const pointer, or find another way to express "none" |
| struct SResource* res_none() { |
| return &kNoResource; |
| } |
| |
| SResource::SResource() |
| : fType(URES_NONE), fWritten(FALSE), fRes(RES_BOGUS), fRes16(-1), fKey(-1), fKey16(-1), |
| line(0), fNext(NULL) { |
| ustr_init(&fComment); |
| } |
| |
| SResource::SResource(SRBRoot *bundle, const char *tag, int8_t type, const UString* comment, |
| UErrorCode &errorCode) |
| : fType(type), fWritten(FALSE), fRes(RES_BOGUS), fRes16(-1), |
| fKey(bundle != NULL ? bundle->addTag(tag, errorCode) : -1), fKey16(-1), |
| line(0), fNext(NULL) { |
| ustr_init(&fComment); |
| if(comment != NULL) { |
| ustr_cpy(&fComment, comment, &errorCode); |
| } |
| } |
| |
| SResource::~SResource() { |
| ustr_deinit(&fComment); |
| } |
| |
| ContainerResource::~ContainerResource() { |
| SResource *current = fFirst; |
| while (current != NULL) { |
| SResource *next = current->fNext; |
| delete current; |
| current = next; |
| } |
| } |
| |
| TableResource::~TableResource() {} |
| |
| // TODO: clarify that containers adopt new items, even in error cases; use LocalPointer |
| void TableResource::add(SResource *res, int linenumber, UErrorCode &errorCode) { |
| if (U_FAILURE(errorCode) || res == NULL || res == &kNoResource) { |
| return; |
| } |
| |
| /* remember this linenumber to report to the user if there is a duplicate key */ |
| res->line = linenumber; |
| |
| /* here we need to traverse the list */ |
| ++fCount; |
| |
| /* is the list still empty? */ |
| if (fFirst == NULL) { |
| fFirst = res; |
| res->fNext = NULL; |
| return; |
| } |
| |
| const char *resKeyString = fRoot->fKeys + res->fKey; |
| |
| SResource *current = fFirst; |
| |
| SResource *prev = NULL; |
| while (current != NULL) { |
| const char *currentKeyString = fRoot->fKeys + current->fKey; |
| int diff; |
| /* |
| * formatVersion 1: compare key strings in native-charset order |
| * formatVersion 2 and up: compare key strings in ASCII order |
| */ |
| if (gFormatVersion == 1 || U_CHARSET_FAMILY == U_ASCII_FAMILY) { |
| diff = uprv_strcmp(currentKeyString, resKeyString); |
| } else { |
| diff = uprv_compareInvCharsAsAscii(currentKeyString, resKeyString); |
| } |
| if (diff < 0) { |
| prev = current; |
| current = current->fNext; |
| } else if (diff > 0) { |
| /* we're either in front of the list, or in the middle */ |
| if (prev == NULL) { |
| /* front of the list */ |
| fFirst = res; |
| } else { |
| /* middle of the list */ |
| prev->fNext = res; |
| } |
| |
| res->fNext = current; |
| return; |
| } else { |
| /* Key already exists! ERROR! */ |
| error(linenumber, "duplicate key '%s' in table, first appeared at line %d", currentKeyString, current->line); |
| errorCode = U_UNSUPPORTED_ERROR; |
| return; |
| } |
| } |
| |
| /* end of list */ |
| prev->fNext = res; |
| res->fNext = NULL; |
| } |
| |
| ArrayResource::~ArrayResource() {} |
| |
| void ArrayResource::add(SResource *res) { |
| if (res != NULL && res != &kNoResource) { |
| if (fFirst == NULL) { |
| fFirst = res; |
| } else { |
| fLast->fNext = res; |
| } |
| fLast = res; |
| ++fCount; |
| } |
| } |
| |
| PseudoListResource::~PseudoListResource() {} |
| |
| void PseudoListResource::add(SResource *res) { |
| if (res != NULL && res != &kNoResource) { |
| res->fNext = fFirst; |
| fFirst = res; |
| ++fCount; |
| } |
| } |
| |
| StringBaseResource::StringBaseResource(SRBRoot *bundle, const char *tag, int8_t type, |
| const UChar *value, int32_t len, |
| const UString* comment, UErrorCode &errorCode) |
| : SResource(bundle, tag, type, comment, errorCode) { |
| if (len == 0 && gFormatVersion > 1) { |
| fRes = URES_MAKE_EMPTY_RESOURCE(type); |
| fWritten = TRUE; |
| return; |
| } |
| |
| fString.setTo(ConstChar16Ptr(value), len); |
| fString.getTerminatedBuffer(); // Some code relies on NUL-termination. |
| if (U_SUCCESS(errorCode) && fString.isBogus()) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| } |
| } |
| |
| StringBaseResource::StringBaseResource(SRBRoot *bundle, int8_t type, |
| const icu::UnicodeString &value, UErrorCode &errorCode) |
| : SResource(bundle, NULL, type, NULL, errorCode), fString(value) { |
| if (value.isEmpty() && gFormatVersion > 1) { |
| fRes = URES_MAKE_EMPTY_RESOURCE(type); |
| fWritten = TRUE; |
| return; |
| } |
| |
| fString.getTerminatedBuffer(); // Some code relies on NUL-termination. |
| if (U_SUCCESS(errorCode) && fString.isBogus()) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| } |
| } |
| |
| // Pool bundle string, alias the buffer. Guaranteed NUL-terminated and not empty. |
| StringBaseResource::StringBaseResource(int8_t type, const UChar *value, int32_t len, |
| UErrorCode &errorCode) |
| : SResource(NULL, NULL, type, NULL, errorCode), fString(TRUE, value, len) { |
| assert(len > 0); |
| assert(!fString.isBogus()); |
| } |
| |
| StringBaseResource::~StringBaseResource() {} |
| |
| static int32_t U_CALLCONV |
| string_hash(const UElement key) { |
| const StringResource *res = static_cast<const StringResource *>(key.pointer); |
| return res->fString.hashCode(); |
| } |
| |
| static UBool U_CALLCONV |
| string_comp(const UElement key1, const UElement key2) { |
| const StringResource *res1 = static_cast<const StringResource *>(key1.pointer); |
| const StringResource *res2 = static_cast<const StringResource *>(key2.pointer); |
| return res1->fString == res2->fString; |
| } |
| |
| StringResource::~StringResource() {} |
| |
| AliasResource::~AliasResource() {} |
| |
| IntResource::IntResource(SRBRoot *bundle, const char *tag, int32_t value, |
| const UString* comment, UErrorCode &errorCode) |
| : SResource(bundle, tag, URES_INT, comment, errorCode) { |
| fValue = value; |
| fRes = URES_MAKE_RESOURCE(URES_INT, value & RES_MAX_OFFSET); |
| fWritten = TRUE; |
| } |
| |
| IntResource::~IntResource() {} |
| |
| IntVectorResource::IntVectorResource(SRBRoot *bundle, const char *tag, |
| const UString* comment, UErrorCode &errorCode) |
| : SResource(bundle, tag, URES_INT_VECTOR, comment, errorCode), |
| fCount(0), fArray(new uint32_t[RESLIST_MAX_INT_VECTOR]) { |
| if (fArray == NULL) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| } |
| |
| IntVectorResource::~IntVectorResource() { |
| delete[] fArray; |
| } |
| |
| void IntVectorResource::add(int32_t value, UErrorCode &errorCode) { |
| if (U_SUCCESS(errorCode)) { |
| fArray[fCount++] = value; |
| } |
| } |
| |
| BinaryResource::BinaryResource(SRBRoot *bundle, const char *tag, |
| uint32_t length, uint8_t *data, const char* fileName, |
| const UString* comment, UErrorCode &errorCode) |
| : SResource(bundle, tag, URES_BINARY, comment, errorCode), |
| fLength(length), fData(NULL), fFileName(NULL) { |
| if (U_FAILURE(errorCode)) { |
| return; |
| } |
| if (fileName != NULL && *fileName != 0){ |
| fFileName = new char[uprv_strlen(fileName)+1]; |
| if (fFileName == NULL) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| uprv_strcpy(fFileName, fileName); |
| } |
| if (length > 0) { |
| fData = new uint8_t[length]; |
| if (fData == NULL) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| uprv_memcpy(fData, data, length); |
| } else { |
| if (gFormatVersion > 1) { |
| fRes = URES_MAKE_EMPTY_RESOURCE(URES_BINARY); |
| fWritten = TRUE; |
| } |
| } |
| } |
| |
| BinaryResource::~BinaryResource() { |
| delete[] fData; |
| delete[] fFileName; |
| } |
| |
| /* Writing Functions */ |
| |
| void |
| StringResource::handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet, |
| UErrorCode &errorCode) { |
| assert(fSame == NULL); |
| fSame = static_cast<StringResource *>(uhash_get(stringSet, this)); |
| if (fSame != NULL) { |
| // This is a duplicate of a pool bundle string or of an earlier-visited string. |
| if (++fSame->fNumCopies == 1) { |
| assert(fSame->fWritten); |
| int32_t poolStringIndex = (int32_t)RES_GET_OFFSET(fSame->fRes); |
| if (poolStringIndex >= bundle->fPoolStringIndexLimit) { |
| bundle->fPoolStringIndexLimit = poolStringIndex + 1; |
| } |
| } |
| return; |
| } |
| /* Put this string into the set for finding duplicates. */ |
| fNumCopies = 1; |
| uhash_put(stringSet, this, this, &errorCode); |
| |
| if (bundle->fStringsForm != STRINGS_UTF16_V1) { |
| int32_t len = length(); |
| if (len <= MAX_IMPLICIT_STRING_LENGTH && |
| !U16_IS_TRAIL(fString[0]) && fString.indexOf((UChar)0) < 0) { |
| /* |
| * This string will be stored without an explicit length. |
| * Runtime will detect !U16_IS_TRAIL(s[0]) and call u_strlen(). |
| */ |
| fNumCharsForLength = 0; |
| } else if (len <= 0x3ee) { |
| fNumCharsForLength = 1; |
| } else if (len <= 0xfffff) { |
| fNumCharsForLength = 2; |
| } else { |
| fNumCharsForLength = 3; |
| } |
| bundle->f16BitStringsLength += fNumCharsForLength + len + 1; /* +1 for the NUL */ |
| } |
| } |
| |
| void |
| ContainerResource::handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet, |
| UErrorCode &errorCode) { |
| for (SResource *current = fFirst; current != NULL; current = current->fNext) { |
| current->preflightStrings(bundle, stringSet, errorCode); |
| } |
| } |
| |
| void |
| SResource::preflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode) { |
| if (U_FAILURE(errorCode)) { |
| return; |
| } |
| if (fRes != RES_BOGUS) { |
| /* |
| * The resource item word was already precomputed, which means |
| * no further data needs to be written. |
| * This might be an integer, or an empty string/binary/etc. |
| */ |
| return; |
| } |
| handlePreflightStrings(bundle, stringSet, errorCode); |
| } |
| |
| void |
| SResource::handlePreflightStrings(SRBRoot * /*bundle*/, UHashtable * /*stringSet*/, |
| UErrorCode & /*errorCode*/) { |
| /* Neither a string nor a container. */ |
| } |
| |
| int32_t |
| SRBRoot::makeRes16(uint32_t resWord) const { |
| if (resWord == 0) { |
| return 0; /* empty string */ |
| } |
| uint32_t type = RES_GET_TYPE(resWord); |
| int32_t offset = (int32_t)RES_GET_OFFSET(resWord); |
| if (type == URES_STRING_V2) { |
| assert(offset > 0); |
| if (offset < fPoolStringIndexLimit) { |
| if (offset < fPoolStringIndex16Limit) { |
| return offset; |
| } |
| } else { |
| offset = offset - fPoolStringIndexLimit + fPoolStringIndex16Limit; |
| if (offset <= 0xffff) { |
| return offset; |
| } |
| } |
| } |
| return -1; |
| } |
| |
| int32_t |
| SRBRoot::mapKey(int32_t oldpos) const { |
| const KeyMapEntry *map = fKeyMap; |
| if (map == NULL) { |
| return oldpos; |
| } |
| int32_t i, start, limit; |
| |
| /* do a binary search for the old, pre-compactKeys() key offset */ |
| start = fUsePoolBundle->fKeysCount; |
| limit = start + fKeysCount; |
| while (start < limit - 1) { |
| i = (start + limit) / 2; |
| if (oldpos < map[i].oldpos) { |
| limit = i; |
| } else { |
| start = i; |
| } |
| } |
| assert(oldpos == map[start].oldpos); |
| return map[start].newpos; |
| } |
| |
| /* |
| * Only called for UTF-16 v1 strings and duplicate UTF-16 v2 strings. |
| * For unique UTF-16 v2 strings, write16() sees fRes != RES_BOGUS |
| * and exits early. |
| */ |
| void |
| StringResource::handleWrite16(SRBRoot * /*bundle*/) { |
| SResource *same; |
| if ((same = fSame) != NULL) { |
| /* This is a duplicate. */ |
| assert(same->fRes != RES_BOGUS && same->fWritten); |
| fRes = same->fRes; |
| fWritten = same->fWritten; |
| } |
| } |
| |
| void |
| ContainerResource::writeAllRes16(SRBRoot *bundle) { |
| for (SResource *current = fFirst; current != NULL; current = current->fNext) { |
| bundle->f16BitUnits.append((UChar)current->fRes16); |
| } |
| fWritten = TRUE; |
| } |
| |
| void |
| ArrayResource::handleWrite16(SRBRoot *bundle) { |
| if (fCount == 0 && gFormatVersion > 1) { |
| fRes = URES_MAKE_EMPTY_RESOURCE(URES_ARRAY); |
| fWritten = TRUE; |
| return; |
| } |
| |
| int32_t res16 = 0; |
| for (SResource *current = fFirst; current != NULL; current = current->fNext) { |
| current->write16(bundle); |
| res16 |= current->fRes16; |
| } |
| if (fCount <= 0xffff && res16 >= 0 && gFormatVersion > 1) { |
| fRes = URES_MAKE_RESOURCE(URES_ARRAY16, bundle->f16BitUnits.length()); |
| bundle->f16BitUnits.append((UChar)fCount); |
| writeAllRes16(bundle); |
| } |
| } |
| |
| void |
| TableResource::handleWrite16(SRBRoot *bundle) { |
| if (fCount == 0 && gFormatVersion > 1) { |
| fRes = URES_MAKE_EMPTY_RESOURCE(URES_TABLE); |
| fWritten = TRUE; |
| return; |
| } |
| /* Find the smallest table type that fits the data. */ |
| int32_t key16 = 0; |
| int32_t res16 = 0; |
| for (SResource *current = fFirst; current != NULL; current = current->fNext) { |
| current->write16(bundle); |
| key16 |= current->fKey16; |
| res16 |= current->fRes16; |
| } |
| if(fCount > (uint32_t)bundle->fMaxTableLength) { |
| bundle->fMaxTableLength = fCount; |
| } |
| if (fCount <= 0xffff && key16 >= 0) { |
| if (res16 >= 0 && gFormatVersion > 1) { |
| /* 16-bit count, key offsets and values */ |
| fRes = URES_MAKE_RESOURCE(URES_TABLE16, bundle->f16BitUnits.length()); |
| bundle->f16BitUnits.append((UChar)fCount); |
| for (SResource *current = fFirst; current != NULL; current = current->fNext) { |
| bundle->f16BitUnits.append((UChar)current->fKey16); |
| } |
| writeAllRes16(bundle); |
| } else { |
| /* 16-bit count, 16-bit key offsets, 32-bit values */ |
| fTableType = URES_TABLE; |
| } |
| } else { |
| /* 32-bit count, key offsets and values */ |
| fTableType = URES_TABLE32; |
| } |
| } |
| |
| void |
| PseudoListResource::handleWrite16(SRBRoot * /*bundle*/) { |
| fRes = URES_MAKE_EMPTY_RESOURCE(URES_TABLE); |
| fWritten = TRUE; |
| } |
| |
| void |
| SResource::write16(SRBRoot *bundle) { |
| if (fKey >= 0) { |
| // A tagged resource has a non-negative key index into the parsed key strings. |
| // compactKeys() built a map from parsed key index to the final key index. |
| // After the mapping, negative key indexes are used for shared pool bundle keys. |
| fKey = bundle->mapKey(fKey); |
| // If the key index fits into a Key16 for a Table or Table16, |
| // then set the fKey16 field accordingly. |
| // Otherwise keep it at -1. |
| if (fKey >= 0) { |
| if (fKey < bundle->fLocalKeyLimit) { |
| fKey16 = fKey; |
| } |
| } else { |
| int32_t poolKeyIndex = fKey & 0x7fffffff; |
| if (poolKeyIndex <= 0xffff) { |
| poolKeyIndex += bundle->fLocalKeyLimit; |
| if (poolKeyIndex <= 0xffff) { |
| fKey16 = poolKeyIndex; |
| } |
| } |
| } |
| } |
| /* |
| * fRes != RES_BOGUS: |
| * The resource item word was already precomputed, which means |
| * no further data needs to be written. |
| * This might be an integer, or an empty or UTF-16 v2 string, |
| * an empty binary, etc. |
| */ |
| if (fRes == RES_BOGUS) { |
| handleWrite16(bundle); |
| } |
| // Compute fRes16 for precomputed as well as just-computed fRes. |
| fRes16 = bundle->makeRes16(fRes); |
| } |
| |
| void |
| SResource::handleWrite16(SRBRoot * /*bundle*/) { |
| /* Only a few resource types write 16-bit units. */ |
| } |
| |
| /* |
| * Only called for UTF-16 v1 strings, and for aliases. |
| * For UTF-16 v2 strings, preWrite() sees fRes != RES_BOGUS |
| * and exits early. |
| */ |
| void |
| StringBaseResource::handlePreWrite(uint32_t *byteOffset) { |
| /* Write the UTF-16 v1 string. */ |
| fRes = URES_MAKE_RESOURCE(fType, *byteOffset >> 2); |
| *byteOffset += 4 + (length() + 1) * U_SIZEOF_UCHAR; |
| } |
| |
| void |
| IntVectorResource::handlePreWrite(uint32_t *byteOffset) { |
| if (fCount == 0 && gFormatVersion > 1) { |
| fRes = URES_MAKE_EMPTY_RESOURCE(URES_INT_VECTOR); |
| fWritten = TRUE; |
| } else { |
| fRes = URES_MAKE_RESOURCE(URES_INT_VECTOR, *byteOffset >> 2); |
| *byteOffset += (1 + fCount) * 4; |
| } |
| } |
| |
| void |
| BinaryResource::handlePreWrite(uint32_t *byteOffset) { |
| uint32_t pad = 0; |
| uint32_t dataStart = *byteOffset + sizeof(fLength); |
| |
| if (dataStart % BIN_ALIGNMENT) { |
| pad = (BIN_ALIGNMENT - dataStart % BIN_ALIGNMENT); |
| *byteOffset += pad; /* pad == 4 or 8 or 12 */ |
| } |
| fRes = URES_MAKE_RESOURCE(URES_BINARY, *byteOffset >> 2); |
| *byteOffset += 4 + fLength; |
| } |
| |
| void |
| ContainerResource::preWriteAllRes(uint32_t *byteOffset) { |
| for (SResource *current = fFirst; current != NULL; current = current->fNext) { |
| current->preWrite(byteOffset); |
| } |
| } |
| |
| void |
| ArrayResource::handlePreWrite(uint32_t *byteOffset) { |
| preWriteAllRes(byteOffset); |
| fRes = URES_MAKE_RESOURCE(URES_ARRAY, *byteOffset >> 2); |
| *byteOffset += (1 + fCount) * 4; |
| } |
| |
| void |
| TableResource::handlePreWrite(uint32_t *byteOffset) { |
| preWriteAllRes(byteOffset); |
| if (fTableType == URES_TABLE) { |
| /* 16-bit count, 16-bit key offsets, 32-bit values */ |
| fRes = URES_MAKE_RESOURCE(URES_TABLE, *byteOffset >> 2); |
| *byteOffset += 2 + fCount * 6; |
| } else { |
| /* 32-bit count, key offsets and values */ |
| fRes = URES_MAKE_RESOURCE(URES_TABLE32, *byteOffset >> 2); |
| *byteOffset += 4 + fCount * 8; |
| } |
| } |
| |
| void |
| SResource::preWrite(uint32_t *byteOffset) { |
| if (fRes != RES_BOGUS) { |
| /* |
| * The resource item word was already precomputed, which means |
| * no further data needs to be written. |
| * This might be an integer, or an empty or UTF-16 v2 string, |
| * an empty binary, etc. |
| */ |
| return; |
| } |
| handlePreWrite(byteOffset); |
| *byteOffset += calcPadding(*byteOffset); |
| } |
| |
| void |
| SResource::handlePreWrite(uint32_t * /*byteOffset*/) { |
| assert(FALSE); |
| } |
| |
| /* |
| * Only called for UTF-16 v1 strings, and for aliases. For UTF-16 v2 strings, |
| * write() sees fWritten and exits early. |
| */ |
| void |
| StringBaseResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) { |
| /* Write the UTF-16 v1 string. */ |
| int32_t len = length(); |
| udata_write32(mem, len); |
| udata_writeUString(mem, getBuffer(), len + 1); |
| *byteOffset += 4 + (len + 1) * U_SIZEOF_UCHAR; |
| fWritten = TRUE; |
| } |
| |
| void |
| ContainerResource::writeAllRes(UNewDataMemory *mem, uint32_t *byteOffset) { |
| uint32_t i = 0; |
| for (SResource *current = fFirst; current != NULL; ++i, current = current->fNext) { |
| current->write(mem, byteOffset); |
| } |
| assert(i == fCount); |
| } |
| |
| void |
| ContainerResource::writeAllRes32(UNewDataMemory *mem, uint32_t *byteOffset) { |
| for (SResource *current = fFirst; current != NULL; current = current->fNext) { |
| udata_write32(mem, current->fRes); |
| } |
| *byteOffset += fCount * 4; |
| } |
| |
| void |
| ArrayResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) { |
| writeAllRes(mem, byteOffset); |
| udata_write32(mem, fCount); |
| *byteOffset += 4; |
| writeAllRes32(mem, byteOffset); |
| } |
| |
| void |
| IntVectorResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) { |
| udata_write32(mem, fCount); |
| for(uint32_t i = 0; i < fCount; ++i) { |
| udata_write32(mem, fArray[i]); |
| } |
| *byteOffset += (1 + fCount) * 4; |
| } |
| |
| void |
| BinaryResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) { |
| uint32_t pad = 0; |
| uint32_t dataStart = *byteOffset + sizeof(fLength); |
| |
| if (dataStart % BIN_ALIGNMENT) { |
| pad = (BIN_ALIGNMENT - dataStart % BIN_ALIGNMENT); |
| udata_writePadding(mem, pad); /* pad == 4 or 8 or 12 */ |
| *byteOffset += pad; |
| } |
| |
| udata_write32(mem, fLength); |
| if (fLength > 0) { |
| udata_writeBlock(mem, fData, fLength); |
| } |
| *byteOffset += 4 + fLength; |
| } |
| |
| void |
| TableResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) { |
| writeAllRes(mem, byteOffset); |
| if(fTableType == URES_TABLE) { |
| udata_write16(mem, (uint16_t)fCount); |
| for (SResource *current = fFirst; current != NULL; current = current->fNext) { |
| udata_write16(mem, current->fKey16); |
| } |
| *byteOffset += (1 + fCount)* 2; |
| if ((fCount & 1) == 0) { |
| /* 16-bit count and even number of 16-bit key offsets need padding before 32-bit resource items */ |
| udata_writePadding(mem, 2); |
| *byteOffset += 2; |
| } |
| } else /* URES_TABLE32 */ { |
| udata_write32(mem, fCount); |
| for (SResource *current = fFirst; current != NULL; current = current->fNext) { |
| udata_write32(mem, (uint32_t)current->fKey); |
| } |
| *byteOffset += (1 + fCount)* 4; |
| } |
| writeAllRes32(mem, byteOffset); |
| } |
| |
| void |
| SResource::write(UNewDataMemory *mem, uint32_t *byteOffset) { |
| if (fWritten) { |
| assert(fRes != RES_BOGUS); |
| return; |
| } |
| handleWrite(mem, byteOffset); |
| uint8_t paddingSize = calcPadding(*byteOffset); |
| if (paddingSize > 0) { |
| udata_writePadding(mem, paddingSize); |
| *byteOffset += paddingSize; |
| } |
| fWritten = TRUE; |
| } |
| |
| void |
| SResource::handleWrite(UNewDataMemory * /*mem*/, uint32_t * /*byteOffset*/) { |
| assert(FALSE); |
| } |
| |
| void SRBRoot::write(const char *outputDir, const char *outputPkg, |
| char *writtenFilename, int writtenFilenameLen, |
| UErrorCode &errorCode) { |
| UNewDataMemory *mem = NULL; |
| uint32_t byteOffset = 0; |
| uint32_t top, size; |
| char dataName[1024]; |
| int32_t indexes[URES_INDEX_TOP]; |
| |
| compactKeys(errorCode); |
| /* |
| * Add padding bytes to fKeys so that fKeysTop is 4-aligned. |
| * Safe because the capacity is a multiple of 4. |
| */ |
| while (fKeysTop & 3) { |
| fKeys[fKeysTop++] = (char)0xaa; |
| } |
| /* |
| * In URES_TABLE, use all local key offsets that fit into 16 bits, |
| * and use the remaining 16-bit offsets for pool key offsets |
| * if there are any. |
| * If there are no local keys, then use the whole 16-bit space |
| * for pool key offsets. |
| * Note: This cannot be changed without changing the major formatVersion. |
| */ |
| if (fKeysBottom < fKeysTop) { |
| if (fKeysTop <= 0x10000) { |
| fLocalKeyLimit = fKeysTop; |
| } else { |
| fLocalKeyLimit = 0x10000; |
| } |
| } else { |
| fLocalKeyLimit = 0; |
| } |
| |
| UHashtable *stringSet; |
| if (gFormatVersion > 1) { |
| stringSet = uhash_open(string_hash, string_comp, string_comp, &errorCode); |
| if (U_SUCCESS(errorCode) && |
| fUsePoolBundle != NULL && fUsePoolBundle->fStrings != NULL) { |
| for (SResource *current = fUsePoolBundle->fStrings->fFirst; |
| current != NULL; |
| current = current->fNext) { |
| StringResource *sr = static_cast<StringResource *>(current); |
| sr->fNumCopies = 0; |
| sr->fNumUnitsSaved = 0; |
| uhash_put(stringSet, sr, sr, &errorCode); |
| } |
| } |
| fRoot->preflightStrings(this, stringSet, errorCode); |
| } else { |
| stringSet = NULL; |
| } |
| if (fStringsForm == STRINGS_UTF16_V2 && f16BitStringsLength > 0) { |
| compactStringsV2(stringSet, errorCode); |
| } |
| uhash_close(stringSet); |
| if (U_FAILURE(errorCode)) { |
| return; |
| } |
| |
| int32_t formatVersion = gFormatVersion; |
| if (fPoolStringIndexLimit != 0) { |
| int32_t sum = fPoolStringIndexLimit + fLocalStringIndexLimit; |
| if ((sum - 1) > RES_MAX_OFFSET) { |
| errorCode = U_BUFFER_OVERFLOW_ERROR; |
| return; |
| } |
| if (fPoolStringIndexLimit < 0x10000 && sum <= 0x10000) { |
| // 16-bit indexes work for all pool + local strings. |
| fPoolStringIndex16Limit = fPoolStringIndexLimit; |
| } else { |
| // Set the pool index threshold so that 16-bit indexes work |
| // for some pool strings and some local strings. |
| fPoolStringIndex16Limit = (int32_t)( |
| ((int64_t)fPoolStringIndexLimit * 0xffff) / sum); |
| } |
| } else if (gIsDefaultFormatVersion && formatVersion == 3 && !fIsPoolBundle) { |
| // If we just default to formatVersion 3 |
| // but there are no pool bundle strings to share |
| // and we do not write a pool bundle, |
| // then write formatVersion 2 which is just as good. |
| formatVersion = 2; |
| } |
| |
| fRoot->write16(this); |
| if (f16BitUnits.isBogus()) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| if (f16BitUnits.length() & 1) { |
| f16BitUnits.append((UChar)0xaaaa); /* pad to multiple of 4 bytes */ |
| } |
| |
| byteOffset = fKeysTop + f16BitUnits.length() * 2; |
| fRoot->preWrite(&byteOffset); |
| |
| /* total size including the root item */ |
| top = byteOffset; |
| |
| if (writtenFilename && writtenFilenameLen) { |
| *writtenFilename = 0; |
| } |
| |
| if (writtenFilename) { |
| int32_t off = 0, len = 0; |
| if (outputDir) { |
| len = (int32_t)uprv_strlen(outputDir); |
| if (len > writtenFilenameLen) { |
| len = writtenFilenameLen; |
| } |
| uprv_strncpy(writtenFilename, outputDir, len); |
| } |
| if (writtenFilenameLen -= len) { |
| off += len; |
| writtenFilename[off] = U_FILE_SEP_CHAR; |
| if (--writtenFilenameLen) { |
| ++off; |
| if(outputPkg != NULL) |
| { |
| uprv_strcpy(writtenFilename+off, outputPkg); |
| off += (int32_t)uprv_strlen(outputPkg); |
| writtenFilename[off] = '_'; |
| ++off; |
| } |
| |
| len = (int32_t)uprv_strlen(fLocale); |
| if (len > writtenFilenameLen) { |
| len = writtenFilenameLen; |
| } |
| uprv_strncpy(writtenFilename + off, fLocale, len); |
| if (writtenFilenameLen -= len) { |
| off += len; |
| len = 5; |
| if (len > writtenFilenameLen) { |
| len = writtenFilenameLen; |
| } |
| uprv_strncpy(writtenFilename + off, ".res", len); |
| } |
| } |
| } |
| } |
| |
| if(outputPkg) |
| { |
| uprv_strcpy(dataName, outputPkg); |
| uprv_strcat(dataName, "_"); |
| uprv_strcat(dataName, fLocale); |
| } |
| else |
| { |
| uprv_strcpy(dataName, fLocale); |
| } |
| |
| uprv_memcpy(dataInfo.formatVersion, gFormatVersions + formatVersion, sizeof(UVersionInfo)); |
| |
| mem = udata_create(outputDir, "res", dataName, |
| &dataInfo, (gIncludeCopyright==TRUE)? U_COPYRIGHT_STRING:NULL, &errorCode); |
| if(U_FAILURE(errorCode)){ |
| return; |
| } |
| |
| /* write the root item */ |
| udata_write32(mem, fRoot->fRes); |
| |
| /* |
| * formatVersion 1.1 (ICU 2.8): |
| * write int32_t indexes[] after root and before the key strings |
| * to make it easier to parse resource bundles in icuswap or from Java etc. |
| */ |
| uprv_memset(indexes, 0, sizeof(indexes)); |
| indexes[URES_INDEX_LENGTH]= fIndexLength; |
| indexes[URES_INDEX_KEYS_TOP]= fKeysTop>>2; |
| indexes[URES_INDEX_RESOURCES_TOP]= (int32_t)(top>>2); |
| indexes[URES_INDEX_BUNDLE_TOP]= indexes[URES_INDEX_RESOURCES_TOP]; |
| indexes[URES_INDEX_MAX_TABLE_LENGTH]= fMaxTableLength; |
| |
| /* |
| * formatVersion 1.2 (ICU 3.6): |
| * write indexes[URES_INDEX_ATTRIBUTES] with URES_ATT_NO_FALLBACK set or not set |
| * the memset() above initialized all indexes[] to 0 |
| */ |
| if (fNoFallback) { |
| indexes[URES_INDEX_ATTRIBUTES]=URES_ATT_NO_FALLBACK; |
| } |
| /* |
| * formatVersion 2.0 (ICU 4.4): |
| * more compact string value storage, optional pool bundle |
| */ |
| if (URES_INDEX_16BIT_TOP < fIndexLength) { |
| indexes[URES_INDEX_16BIT_TOP] = (fKeysTop>>2) + (f16BitUnits.length()>>1); |
| } |
| if (URES_INDEX_POOL_CHECKSUM < fIndexLength) { |
| if (fIsPoolBundle) { |
| indexes[URES_INDEX_ATTRIBUTES] |= URES_ATT_IS_POOL_BUNDLE | URES_ATT_NO_FALLBACK; |
| uint32_t checksum = computeCRC((const char *)(fKeys + fKeysBottom), |
| (uint32_t)(fKeysTop - fKeysBottom), 0); |
| if (f16BitUnits.length() <= 1) { |
| // no pool strings to checksum |
| } else if (U_IS_BIG_ENDIAN) { |
| checksum = computeCRC(reinterpret_cast<const char *>(f16BitUnits.getBuffer()), |
| (uint32_t)f16BitUnits.length() * 2, checksum); |
| } else { |
| // Swap to big-endian so we get the same checksum on all platforms |
| // (except for charset family, due to the key strings). |
| UnicodeString s(f16BitUnits); |
| assert(!s.isBogus()); |
| // .getBuffer(capacity) returns a mutable buffer |
| char16_t* p = s.getBuffer(f16BitUnits.length()); |
| for (int32_t count = f16BitUnits.length(); count > 0; --count) { |
| uint16_t x = *p; |
| *p++ = (uint16_t)((x << 8) | (x >> 8)); |
| } |
| s.releaseBuffer(f16BitUnits.length()); |
| checksum = computeCRC((const char *)s.getBuffer(), |
| (uint32_t)f16BitUnits.length() * 2, checksum); |
| } |
| indexes[URES_INDEX_POOL_CHECKSUM] = (int32_t)checksum; |
| } else if (gUsePoolBundle) { |
| indexes[URES_INDEX_ATTRIBUTES] |= URES_ATT_USES_POOL_BUNDLE; |
| indexes[URES_INDEX_POOL_CHECKSUM] = fUsePoolBundle->fChecksum; |
| } |
| } |
| // formatVersion 3 (ICU 56): |
| // share string values via pool bundle strings |
| indexes[URES_INDEX_LENGTH] |= fPoolStringIndexLimit << 8; // bits 23..0 -> 31..8 |
| indexes[URES_INDEX_ATTRIBUTES] |= (fPoolStringIndexLimit >> 12) & 0xf000; // bits 27..24 -> 15..12 |
| indexes[URES_INDEX_ATTRIBUTES] |= fPoolStringIndex16Limit << 16; |
| |
| /* write the indexes[] */ |
| udata_writeBlock(mem, indexes, fIndexLength*4); |
| |
| /* write the table key strings */ |
| udata_writeBlock(mem, fKeys+fKeysBottom, |
| fKeysTop-fKeysBottom); |
| |
| /* write the v2 UTF-16 strings, URES_TABLE16 and URES_ARRAY16 */ |
| udata_writeBlock(mem, f16BitUnits.getBuffer(), f16BitUnits.length()*2); |
| |
| /* write all of the bundle contents: the root item and its children */ |
| byteOffset = fKeysTop + f16BitUnits.length() * 2; |
| fRoot->write(mem, &byteOffset); |
| assert(byteOffset == top); |
| |
| size = udata_finish(mem, &errorCode); |
| if(top != size) { |
| fprintf(stderr, "genrb error: wrote %u bytes but counted %u\n", |
| (int)size, (int)top); |
| errorCode = U_INTERNAL_PROGRAM_ERROR; |
| } |
| } |
| |
| /* Opening Functions */ |
| |
| TableResource* table_open(struct SRBRoot *bundle, const char *tag, const struct UString* comment, UErrorCode *status) { |
| LocalPointer<TableResource> res(new TableResource(bundle, tag, comment, *status), *status); |
| return U_SUCCESS(*status) ? res.orphan() : NULL; |
| } |
| |
| ArrayResource* array_open(struct SRBRoot *bundle, const char *tag, const struct UString* comment, UErrorCode *status) { |
| LocalPointer<ArrayResource> res(new ArrayResource(bundle, tag, comment, *status), *status); |
| return U_SUCCESS(*status) ? res.orphan() : NULL; |
| } |
| |
| struct SResource *string_open(struct SRBRoot *bundle, const char *tag, const UChar *value, int32_t len, const struct UString* comment, UErrorCode *status) { |
| LocalPointer<SResource> res( |
| new StringResource(bundle, tag, value, len, comment, *status), *status); |
| return U_SUCCESS(*status) ? res.orphan() : NULL; |
| } |
| |
| struct SResource *alias_open(struct SRBRoot *bundle, const char *tag, UChar *value, int32_t len, const struct UString* comment, UErrorCode *status) { |
| LocalPointer<SResource> res( |
| new AliasResource(bundle, tag, value, len, comment, *status), *status); |
| return U_SUCCESS(*status) ? res.orphan() : NULL; |
| } |
| |
| IntVectorResource *intvector_open(struct SRBRoot *bundle, const char *tag, const struct UString* comment, UErrorCode *status) { |
| LocalPointer<IntVectorResource> res( |
| new IntVectorResource(bundle, tag, comment, *status), *status); |
| return U_SUCCESS(*status) ? res.orphan() : NULL; |
| } |
| |
| struct SResource *int_open(struct SRBRoot *bundle, const char *tag, int32_t value, const struct UString* comment, UErrorCode *status) { |
| LocalPointer<SResource> res(new IntResource(bundle, tag, value, comment, *status), *status); |
| return U_SUCCESS(*status) ? res.orphan() : NULL; |
| } |
| |
| struct SResource *bin_open(struct SRBRoot *bundle, const char *tag, uint32_t length, uint8_t *data, const char* fileName, const struct UString* comment, UErrorCode *status) { |
| LocalPointer<SResource> res( |
| new BinaryResource(bundle, tag, length, data, fileName, comment, *status), *status); |
| return U_SUCCESS(*status) ? res.orphan() : NULL; |
| } |
| |
| SRBRoot::SRBRoot(const UString *comment, UBool isPoolBundle, UErrorCode &errorCode) |
| : fRoot(NULL), fLocale(NULL), fIndexLength(0), fMaxTableLength(0), fNoFallback(FALSE), |
| fStringsForm(STRINGS_UTF16_V1), fIsPoolBundle(isPoolBundle), |
| fKeys(NULL), fKeyMap(NULL), |
| fKeysBottom(0), fKeysTop(0), fKeysCapacity(0), |
| fKeysCount(0), fLocalKeyLimit(0), |
| f16BitUnits(), f16BitStringsLength(0), |
| fUsePoolBundle(&kNoPoolBundle), |
| fPoolStringIndexLimit(0), fPoolStringIndex16Limit(0), fLocalStringIndexLimit(0), |
| fWritePoolBundle(NULL) { |
| if (U_FAILURE(errorCode)) { |
| return; |
| } |
| |
| if (gFormatVersion > 1) { |
| // f16BitUnits must start with a zero for empty resources. |
| // We might be able to omit it if there are no empty 16-bit resources. |
| f16BitUnits.append((UChar)0); |
| } |
| |
| fKeys = (char *) uprv_malloc(sizeof(char) * KEY_SPACE_SIZE); |
| if (isPoolBundle) { |
| fRoot = new PseudoListResource(this, errorCode); |
| } else { |
| fRoot = new TableResource(this, NULL, comment, errorCode); |
| } |
| if (fKeys == NULL || fRoot == NULL || U_FAILURE(errorCode)) { |
| if (U_SUCCESS(errorCode)) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| } |
| return; |
| } |
| |
| fKeysCapacity = KEY_SPACE_SIZE; |
| /* formatVersion 1.1 and up: start fKeysTop after the root item and indexes[] */ |
| if (gUsePoolBundle || isPoolBundle) { |
| fIndexLength = URES_INDEX_POOL_CHECKSUM + 1; |
| } else if (gFormatVersion >= 2) { |
| fIndexLength = URES_INDEX_16BIT_TOP + 1; |
| } else /* formatVersion 1 */ { |
| fIndexLength = URES_INDEX_ATTRIBUTES + 1; |
| } |
| fKeysBottom = (1 /* root */ + fIndexLength) * 4; |
| uprv_memset(fKeys, 0, fKeysBottom); |
| fKeysTop = fKeysBottom; |
| |
| if (gFormatVersion == 1) { |
| fStringsForm = STRINGS_UTF16_V1; |
| } else { |
| fStringsForm = STRINGS_UTF16_V2; |
| } |
| } |
| |
| /* Closing Functions */ |
| |
| void res_close(struct SResource *res) { |
| delete res; |
| } |
| |
| SRBRoot::~SRBRoot() { |
| delete fRoot; |
| uprv_free(fLocale); |
| uprv_free(fKeys); |
| uprv_free(fKeyMap); |
| } |
| |
| /* Misc Functions */ |
| |
| void SRBRoot::setLocale(UChar *locale, UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode)) { |
| return; |
| } |
| |
| uprv_free(fLocale); |
| fLocale = (char*) uprv_malloc(sizeof(char) * (u_strlen(locale)+1)); |
| if(fLocale == NULL) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| |
| u_UCharsToChars(locale, fLocale, u_strlen(locale)+1); |
| } |
| |
| const char * |
| SRBRoot::getKeyString(int32_t key) const { |
| if (key < 0) { |
| return fUsePoolBundle->fKeys + (key & 0x7fffffff); |
| } else { |
| return fKeys + key; |
| } |
| } |
| |
| const char * |
| SResource::getKeyString(const SRBRoot *bundle) const { |
| if (fKey == -1) { |
| return NULL; |
| } |
| return bundle->getKeyString(fKey); |
| } |
| |
| const char * |
| SRBRoot::getKeyBytes(int32_t *pLength) const { |
| *pLength = fKeysTop - fKeysBottom; |
| return fKeys + fKeysBottom; |
| } |
| |
| int32_t |
| SRBRoot::addKeyBytes(const char *keyBytes, int32_t length, UErrorCode &errorCode) { |
| int32_t keypos; |
| |
| // It is not legal to add new key bytes after compactKeys is run! |
| U_ASSERT(fKeyMap == nullptr); |
| |
| if (U_FAILURE(errorCode)) { |
| return -1; |
| } |
| if (length < 0 || (keyBytes == NULL && length != 0)) { |
| errorCode = U_ILLEGAL_ARGUMENT_ERROR; |
| return -1; |
| } |
| if (length == 0) { |
| return fKeysTop; |
| } |
| |
| keypos = fKeysTop; |
| fKeysTop += length; |
| if (fKeysTop >= fKeysCapacity) { |
| /* overflow - resize the keys buffer */ |
| fKeysCapacity += KEY_SPACE_SIZE; |
| fKeys = static_cast<char *>(uprv_realloc(fKeys, fKeysCapacity)); |
| if(fKeys == NULL) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| return -1; |
| } |
| } |
| |
| uprv_memcpy(fKeys + keypos, keyBytes, length); |
| |
| return keypos; |
| } |
| |
| int32_t |
| SRBRoot::addTag(const char *tag, UErrorCode &errorCode) { |
| int32_t keypos; |
| |
| if (U_FAILURE(errorCode)) { |
| return -1; |
| } |
| |
| if (tag == NULL) { |
| /* no error: the root table and array items have no keys */ |
| return -1; |
| } |
| |
| keypos = addKeyBytes(tag, (int32_t)(uprv_strlen(tag) + 1), errorCode); |
| if (U_SUCCESS(errorCode)) { |
| ++fKeysCount; |
| } |
| return keypos; |
| } |
| |
| static int32_t |
| compareInt32(int32_t lPos, int32_t rPos) { |
| /* |
| * Compare possibly-negative key offsets. Don't just return lPos - rPos |
| * because that is prone to negative-integer underflows. |
| */ |
| if (lPos < rPos) { |
| return -1; |
| } else if (lPos > rPos) { |
| return 1; |
| } else { |
| return 0; |
| } |
| } |
| |
| static int32_t U_CALLCONV |
| compareKeySuffixes(const void *context, const void *l, const void *r) { |
| const struct SRBRoot *bundle=(const struct SRBRoot *)context; |
| int32_t lPos = ((const KeyMapEntry *)l)->oldpos; |
| int32_t rPos = ((const KeyMapEntry *)r)->oldpos; |
| const char *lStart = bundle->getKeyString(lPos); |
| const char *lLimit = lStart; |
| const char *rStart = bundle->getKeyString(rPos); |
| const char *rLimit = rStart; |
| int32_t diff; |
| while (*lLimit != 0) { ++lLimit; } |
| while (*rLimit != 0) { ++rLimit; } |
| /* compare keys in reverse character order */ |
| while (lStart < lLimit && rStart < rLimit) { |
| diff = (int32_t)(uint8_t)*--lLimit - (int32_t)(uint8_t)*--rLimit; |
| if (diff != 0) { |
| return diff; |
| } |
| } |
| /* sort equal suffixes by descending key length */ |
| diff = (int32_t)(rLimit - rStart) - (int32_t)(lLimit - lStart); |
| if (diff != 0) { |
| return diff; |
| } |
| /* Sort pool bundle keys first (negative oldpos), and otherwise keys in parsing order. */ |
| return compareInt32(lPos, rPos); |
| } |
| |
| static int32_t U_CALLCONV |
| compareKeyNewpos(const void * /*context*/, const void *l, const void *r) { |
| return compareInt32(((const KeyMapEntry *)l)->newpos, ((const KeyMapEntry *)r)->newpos); |
| } |
| |
| static int32_t U_CALLCONV |
| compareKeyOldpos(const void * /*context*/, const void *l, const void *r) { |
| return compareInt32(((const KeyMapEntry *)l)->oldpos, ((const KeyMapEntry *)r)->oldpos); |
| } |
| |
| void SResource::collectKeys(std::function<void(int32_t)> collector) const { |
| collector(fKey); |
| } |
| |
| void ContainerResource::collectKeys(std::function<void(int32_t)> collector) const { |
| collector(fKey); |
| for (SResource* curr = fFirst; curr != NULL; curr = curr->fNext) { |
| curr->collectKeys(collector); |
| } |
| } |
| |
| void |
| SRBRoot::compactKeys(UErrorCode &errorCode) { |
| KeyMapEntry *map; |
| char *keys; |
| int32_t i; |
| |
| // Except for pool bundles, keys might not be used. |
| // Do not add unused keys to the final bundle. |
| std::set<int32_t> keysInUse; |
| if (!fIsPoolBundle) { |
| fRoot->collectKeys([&keysInUse](int32_t key) { |
| if (key >= 0) { |
| keysInUse.insert(key); |
| } |
| }); |
| fKeysCount = static_cast<int32_t>(keysInUse.size()); |
| } |
| |
| int32_t keysCount = fUsePoolBundle->fKeysCount + fKeysCount; |
| if (U_FAILURE(errorCode) || fKeyMap != NULL) { |
| return; |
| } |
| map = (KeyMapEntry *)uprv_malloc(keysCount * sizeof(KeyMapEntry)); |
| if (map == NULL) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| keys = (char *)fUsePoolBundle->fKeys; |
| for (i = 0; i < fUsePoolBundle->fKeysCount; ++i) { |
| map[i].oldpos = |
| (int32_t)(keys - fUsePoolBundle->fKeys) | 0x80000000; /* negative oldpos */ |
| map[i].newpos = 0; |
| while (*keys != 0) { ++keys; } /* skip the key */ |
| ++keys; /* skip the NUL */ |
| } |
| keys = fKeys + fKeysBottom; |
| while (i < keysCount) { |
| int32_t keyOffset = static_cast<int32_t>(keys - fKeys); |
| if (!fIsPoolBundle && keysInUse.count(keyOffset) == 0) { |
| // Mark the unused key as deleted |
| while (*keys != 0) { *keys++ = 1; } |
| *keys++ = 1; |
| } else { |
| map[i].oldpos = keyOffset; |
| map[i].newpos = 0; |
| while (*keys != 0) { ++keys; } /* skip the key */ |
| ++keys; /* skip the NUL */ |
| i++; |
| } |
| } |
| if (keys != fKeys + fKeysTop) { |
| // Throw away any unused keys from the end |
| fKeysTop = static_cast<int32_t>(keys - fKeys); |
| } |
| /* Sort the keys so that each one is immediately followed by all of its suffixes. */ |
| uprv_sortArray(map, keysCount, (int32_t)sizeof(KeyMapEntry), |
| compareKeySuffixes, this, FALSE, &errorCode); |
| /* |
| * Make suffixes point into earlier, longer strings that contain them |
| * and mark the old, now unused suffix bytes as deleted. |
| */ |
| if (U_SUCCESS(errorCode)) { |
| keys = fKeys; |
| for (i = 0; i < keysCount;) { |
| /* |
| * This key is not a suffix of the previous one; |
| * keep this one and delete the following ones that are |
| * suffixes of this one. |
| */ |
| const char *key; |
| const char *keyLimit; |
| int32_t j = i + 1; |
| map[i].newpos = map[i].oldpos; |
| if (j < keysCount && map[j].oldpos < 0) { |
| /* Key string from the pool bundle, do not delete. */ |
| i = j; |
| continue; |
| } |
| key = getKeyString(map[i].oldpos); |
| for (keyLimit = key; *keyLimit != 0; ++keyLimit) {} |
| for (; j < keysCount && map[j].oldpos >= 0; ++j) { |
| const char *k; |
| char *suffix; |
| const char *suffixLimit; |
| int32_t offset; |
| suffix = keys + map[j].oldpos; |
| for (suffixLimit = suffix; *suffixLimit != 0; ++suffixLimit) {} |
| offset = static_cast<int32_t>((keyLimit - key) - (suffixLimit - suffix)); |
| if (offset < 0) { |
| break; /* suffix cannot be longer than the original */ |
| } |
| /* Is it a suffix of the earlier, longer key? */ |
| for (k = keyLimit; suffix < suffixLimit && *--k == *--suffixLimit;) {} |
| if (suffix == suffixLimit && *k == *suffixLimit) { |
| map[j].newpos = map[i].oldpos + offset; /* yes, point to the earlier key */ |
| // Mark the suffix as deleted |
| while (*suffix != 0) { *suffix++ = 1; } |
| *suffix = 1; |
| } else { |
| break; /* not a suffix, restart from here */ |
| } |
| } |
| i = j; |
| } |
| /* |
| * Re-sort by newpos, then modify the key characters array in-place |
| * to squeeze out unused bytes, and readjust the newpos offsets. |
| */ |
| uprv_sortArray(map, keysCount, (int32_t)sizeof(KeyMapEntry), |
| compareKeyNewpos, NULL, FALSE, &errorCode); |
| if (U_SUCCESS(errorCode)) { |
| int32_t oldpos, newpos, limit; |
| oldpos = newpos = fKeysBottom; |
| limit = fKeysTop; |
| /* skip key offsets that point into the pool bundle rather than this new bundle */ |
| for (i = 0; i < keysCount && map[i].newpos < 0; ++i) {} |
| if (i < keysCount) { |
| while (oldpos < limit) { |
| if (keys[oldpos] == 1) { |
| ++oldpos; /* skip unused bytes */ |
| } else { |
| /* adjust the new offsets for keys starting here */ |
| while (i < keysCount && map[i].newpos == oldpos) { |
| map[i++].newpos = newpos; |
| } |
| /* move the key characters to their new position */ |
| keys[newpos++] = keys[oldpos++]; |
| } |
| } |
| U_ASSERT(i == keysCount); |
| } |
| fKeysTop = newpos; |
| /* Re-sort once more, by old offsets for binary searching. */ |
| uprv_sortArray(map, keysCount, (int32_t)sizeof(KeyMapEntry), |
| compareKeyOldpos, NULL, FALSE, &errorCode); |
| if (U_SUCCESS(errorCode)) { |
| /* key size reduction by limit - newpos */ |
| fKeyMap = map; |
| map = NULL; |
| } |
| } |
| } |
| uprv_free(map); |
| } |
| |
| static int32_t U_CALLCONV |
| compareStringSuffixes(const void * /*context*/, const void *l, const void *r) { |
| const StringResource *left = *((const StringResource **)l); |
| const StringResource *right = *((const StringResource **)r); |
| const UChar *lStart = left->getBuffer(); |
| const UChar *lLimit = lStart + left->length(); |
| const UChar *rStart = right->getBuffer(); |
| const UChar *rLimit = rStart + right->length(); |
| int32_t diff; |
| /* compare keys in reverse character order */ |
| while (lStart < lLimit && rStart < rLimit) { |
| diff = (int32_t)*--lLimit - (int32_t)*--rLimit; |
| if (diff != 0) { |
| return diff; |
| } |
| } |
| /* sort equal suffixes by descending string length */ |
| return right->length() - left->length(); |
| } |
| |
| static int32_t U_CALLCONV |
| compareStringLengths(const void * /*context*/, const void *l, const void *r) { |
| const StringResource *left = *((const StringResource **)l); |
| const StringResource *right = *((const StringResource **)r); |
| int32_t diff; |
| /* Make "is suffix of another string" compare greater than a non-suffix. */ |
| diff = (int)(left->fSame != NULL) - (int)(right->fSame != NULL); |
| if (diff != 0) { |
| return diff; |
| } |
| /* sort by ascending string length */ |
| diff = left->length() - right->length(); |
| if (diff != 0) { |
| return diff; |
| } |
| // sort by descending size reduction |
| diff = right->fNumUnitsSaved - left->fNumUnitsSaved; |
| if (diff != 0) { |
| return diff; |
| } |
| // sort lexically |
| return left->fString.compare(right->fString); |
| } |
| |
| void |
| StringResource::writeUTF16v2(int32_t base, UnicodeString &dest) { |
| int32_t len = length(); |
| fRes = URES_MAKE_RESOURCE(URES_STRING_V2, base + dest.length()); |
| fWritten = TRUE; |
| switch(fNumCharsForLength) { |
| case 0: |
| break; |
| case 1: |
| dest.append((UChar)(0xdc00 + len)); |
| break; |
| case 2: |
| dest.append((UChar)(0xdfef + (len >> 16))); |
| dest.append((UChar)len); |
| break; |
| case 3: |
| dest.append((UChar)0xdfff); |
| dest.append((UChar)(len >> 16)); |
| dest.append((UChar)len); |
| break; |
| default: |
| break; /* will not occur */ |
| } |
| dest.append(fString); |
| dest.append((UChar)0); |
| } |
| |
| void |
| SRBRoot::compactStringsV2(UHashtable *stringSet, UErrorCode &errorCode) { |
| if (U_FAILURE(errorCode)) { |
| return; |
| } |
| // Store the StringResource pointers in an array for |
| // easy sorting and processing. |
| // We enumerate a set of strings, so there are no duplicates. |
| int32_t count = uhash_count(stringSet); |
| LocalArray<StringResource *> array(new StringResource *[count], errorCode); |
| if (U_FAILURE(errorCode)) { |
| return; |
| } |
| for (int32_t pos = UHASH_FIRST, i = 0; i < count; ++i) { |
| array[i] = (StringResource *)uhash_nextElement(stringSet, &pos)->key.pointer; |
| } |
| /* Sort the strings so that each one is immediately followed by all of its suffixes. */ |
| uprv_sortArray(array.getAlias(), count, (int32_t)sizeof(struct SResource **), |
| compareStringSuffixes, NULL, FALSE, &errorCode); |
| if (U_FAILURE(errorCode)) { |
| return; |
| } |
| /* |
| * Make suffixes point into earlier, longer strings that contain them. |
| * Temporarily use fSame and fSuffixOffset for suffix strings to |
| * refer to the remaining ones. |
| */ |
| for (int32_t i = 0; i < count;) { |
| /* |
| * This string is not a suffix of the previous one; |
| * write this one and subsume the following ones that are |
| * suffixes of this one. |
| */ |
| StringResource *res = array[i]; |
| res->fNumUnitsSaved = (res->fNumCopies - 1) * res->get16BitStringsLength(); |
| // Whole duplicates of pool strings are already account for in fPoolStringIndexLimit, |
| // see StringResource::handlePreflightStrings(). |
| int32_t j; |
| for (j = i + 1; j < count; ++j) { |
| StringResource *suffixRes = array[j]; |
| /* Is it a suffix of the earlier, longer string? */ |
| if (res->fString.endsWith(suffixRes->fString)) { |
| assert(res->length() != suffixRes->length()); // Set strings are unique. |
| if (suffixRes->fWritten) { |
| // Pool string, skip. |
| } else if (suffixRes->fNumCharsForLength == 0) { |
| /* yes, point to the earlier string */ |
| suffixRes->fSame = res; |
| suffixRes->fSuffixOffset = res->length() - suffixRes->length(); |
| if (res->fWritten) { |
| // Suffix-share res which is a pool string. |
| // Compute the resource word and collect the maximum. |
| suffixRes->fRes = |
| res->fRes + res->fNumCharsForLength + suffixRes->fSuffixOffset; |
| int32_t poolStringIndex = (int32_t)RES_GET_OFFSET(suffixRes->fRes); |
| if (poolStringIndex >= fPoolStringIndexLimit) { |
| fPoolStringIndexLimit = poolStringIndex + 1; |
| } |
| suffixRes->fWritten = TRUE; |
| } |
| res->fNumUnitsSaved += suffixRes->fNumCopies * suffixRes->get16BitStringsLength(); |
| } else { |
| /* write the suffix by itself if we need explicit length */ |
| } |
| } else { |
| break; /* not a suffix, restart from here */ |
| } |
| } |
| i = j; |
| } |
| /* |
| * Re-sort the strings by ascending length (except suffixes last) |
| * to optimize for URES_TABLE16 and URES_ARRAY16: |
| * Keep as many as possible within reach of 16-bit offsets. |
| */ |
| uprv_sortArray(array.getAlias(), count, (int32_t)sizeof(struct SResource **), |
| compareStringLengths, NULL, FALSE, &errorCode); |
| if (U_FAILURE(errorCode)) { |
| return; |
| } |
| if (fIsPoolBundle) { |
| // Write strings that are sufficiently shared. |
| // Avoid writing other strings. |
| int32_t numStringsWritten = 0; |
| int32_t numUnitsSaved = 0; |
| int32_t numUnitsNotSaved = 0; |
| for (int32_t i = 0; i < count; ++i) { |
| StringResource *res = array[i]; |
| // Maximum pool string index when suffix-sharing the last character. |
| int32_t maxStringIndex = |
| f16BitUnits.length() + res->fNumCharsForLength + res->length() - 1; |
| if (res->fNumUnitsSaved >= GENRB_MIN_16BIT_UNITS_SAVED_FOR_POOL_STRING && |
| maxStringIndex < RES_MAX_OFFSET) { |
| res->writeUTF16v2(0, f16BitUnits); |
| ++numStringsWritten; |
| numUnitsSaved += res->fNumUnitsSaved; |
| } else { |
| numUnitsNotSaved += res->fNumUnitsSaved; |
| res->fRes = URES_MAKE_EMPTY_RESOURCE(URES_STRING); |
| res->fWritten = TRUE; |
| } |
| } |
| if (f16BitUnits.isBogus()) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| } |
| if (getShowWarning()) { // not quiet |
| printf("number of shared strings: %d\n", (int)numStringsWritten); |
| printf("16-bit units for strings: %6d = %6d bytes\n", |
| (int)f16BitUnits.length(), (int)f16BitUnits.length() * 2); |
| printf("16-bit units saved: %6d = %6d bytes\n", |
| (int)numUnitsSaved, (int)numUnitsSaved * 2); |
| printf("16-bit units not saved: %6d = %6d bytes\n", |
| (int)numUnitsNotSaved, (int)numUnitsNotSaved * 2); |
| } |
| } else { |
| assert(fPoolStringIndexLimit <= fUsePoolBundle->fStringIndexLimit); |
| /* Write the non-suffix strings. */ |
| int32_t i; |
| for (i = 0; i < count && array[i]->fSame == NULL; ++i) { |
| StringResource *res = array[i]; |
| if (!res->fWritten) { |
| int32_t localStringIndex = f16BitUnits.length(); |
| if (localStringIndex >= fLocalStringIndexLimit) { |
| fLocalStringIndexLimit = localStringIndex + 1; |
| } |
| res->writeUTF16v2(fPoolStringIndexLimit, f16BitUnits); |
| } |
| } |
| if (f16BitUnits.isBogus()) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| if (fWritePoolBundle != NULL && gFormatVersion >= 3) { |
| PseudoListResource *poolStrings = |
| static_cast<PseudoListResource *>(fWritePoolBundle->fRoot); |
| for (i = 0; i < count && array[i]->fSame == NULL; ++i) { |
| assert(!array[i]->fString.isEmpty()); |
| StringResource *poolString = |
| new StringResource(fWritePoolBundle, array[i]->fString, errorCode); |
| if (poolString == NULL) { |
| errorCode = U_MEMORY_ALLOCATION_ERROR; |
| break; |
| } |
| poolStrings->add(poolString); |
| } |
| } |
| /* Write the suffix strings. Make each point to the real string. */ |
| for (; i < count; ++i) { |
| StringResource *res = array[i]; |
| if (res->fWritten) { |
| continue; |
| } |
| StringResource *same = res->fSame; |
| assert(res->length() != same->length()); // Set strings are unique. |
| res->fRes = same->fRes + same->fNumCharsForLength + res->fSuffixOffset; |
| int32_t localStringIndex = (int32_t)RES_GET_OFFSET(res->fRes) - fPoolStringIndexLimit; |
| // Suffixes of pool strings have been set already. |
| assert(localStringIndex >= 0); |
| if (localStringIndex >= fLocalStringIndexLimit) { |
| fLocalStringIndexLimit = localStringIndex + 1; |
| } |
| res->fWritten = TRUE; |
| } |
| } |
| // +1 to account for the initial zero in f16BitUnits |
| assert(f16BitUnits.length() <= (f16BitStringsLength + 1)); |
| } |
| |
| void SResource::applyFilter( |
| const PathFilter& /*filter*/, |
| ResKeyPath& /*path*/, |
| const SRBRoot* /*bundle*/) { |
| // Only a few resource types (tables) are capable of being filtered. |
| } |
| |
| void TableResource::applyFilter( |
| const PathFilter& filter, |
| ResKeyPath& path, |
| const SRBRoot* bundle) { |
| SResource* prev = nullptr; |
| SResource* curr = fFirst; |
| for (; curr != nullptr;) { |
| path.push(curr->getKeyString(bundle)); |
| auto inclusion = filter.match(path); |
| if (inclusion == PathFilter::EInclusion::INCLUDE) { |
| // Include whole subtree |
| // no-op |
| if (isVerbose()) { |
| std::cout << "genrb subtree: " << bundle->fLocale << ": INCLUDE: " << path << std::endl; |
| } |
| } else if (inclusion == PathFilter::EInclusion::EXCLUDE) { |
| // Reject the whole subtree |
| // Remove it from the linked list |
| if (isVerbose()) { |
| std::cout << "genrb subtree: " << bundle->fLocale << ": DELETE: " << path << std::endl; |
| } |
| if (prev == nullptr) { |
| fFirst = curr->fNext; |
| } else { |
| prev->fNext = curr->fNext; |
| } |
| fCount--; |
| delete curr; |
| curr = prev; |
| } else { |
| U_ASSERT(inclusion == PathFilter::EInclusion::PARTIAL); |
| // Recurse into the child |
| curr->applyFilter(filter, path, bundle); |
| } |
| path.pop(); |
| |
| prev = curr; |
| if (curr == nullptr) { |
| curr = fFirst; |
| } else { |
| curr = curr->fNext; |
| } |
| } |
| } |
