Google Git
Sign in
cobalt / cobalt / 8c0a79f7d6d2292adce752caee02d1bf07bb92f6 / . / src / third_party / mozjs / intl / icu / source / common / ucnvmbcs.c
blob: becb49237e833d6314213044bf2a4587d6246c17 [file] [log] [blame]
/*
******************************************************************************
*
* Copyright (C) 2000-2012, International Business Machines
* Corporation and others. All Rights Reserved.
*
******************************************************************************
* file name: ucnvmbcs.c
* encoding: US-ASCII
* tab size: 8 (not used)
* indentation:4
*
* created on: 2000jul03
* created by: Markus W. Scherer
*
* The current code in this file replaces the previous implementation
* of conversion code from multi-byte codepages to Unicode and back.
* This implementation supports the following:
* - legacy variable-length codepages with up to 4 bytes per character
* - all Unicode code points (up to 0x10ffff)
* - efficient distinction of unassigned vs. illegal byte sequences
* - it is possible in fromUnicode() to directly deal with simple
* stateful encodings (used for EBCDIC_STATEFUL)
* - it is possible to convert Unicode code points
* to a single zero byte (but not as a fallback except for SBCS)
*
* Remaining limitations in fromUnicode:
* - byte sequences must not have leading zero bytes
* - except for SBCS codepages: no fallback mapping from Unicode to a zero byte
* - limitation to up to 4 bytes per character
*
* ICU 2.8 (late 2003) adds a secondary data structure which lifts some of these
* limitations and adds m:n character mappings and other features.
* See ucnv_ext.h for details.
*
* Change history:
*
* 5/6/2001 Ram Moved MBCS_SINGLE_RESULT_FROM_U,MBCS_STAGE_2_FROM_U,
* MBCS_VALUE_2_FROM_STAGE_2, MBCS_VALUE_4_FROM_STAGE_2
* macros to ucnvmbcs.h file
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION
#include "unicode/ucnv.h"
#include "unicode/ucnv_cb.h"
#include "unicode/udata.h"
#include "unicode/uset.h"
#include "unicode/utf8.h"
#include "unicode/utf16.h"
#include "ucnv_bld.h"
#include "ucnvmbcs.h"
#include "ucnv_ext.h"
#include "ucnv_cnv.h"
#include "cmemory.h"
#include "cstring.h"
#include "umutex.h"
/* control optimizations according to the platform */
#define MBCS_UNROLL_SINGLE_TO_BMP 1
#define MBCS_UNROLL_SINGLE_FROM_BMP 0
/*
* _MBCSHeader versions 5.3 & 4.3
* (Note that the _MBCSHeader version is in addition to the converter formatVersion.)
*
* This version is optional. Version 5 is used for incompatible data format changes.
* makeconv will continue to generate version 4 files if possible.
*
* Changes from version 4:
*
* The main difference is an additional _MBCSHeader field with
* - the length (number of uint32_t) of the _MBCSHeader
* - flags for further incompatible data format changes
* - flags for further, backward compatible data format changes
*
* The MBCS_OPT_FROM_U flag indicates that most of the fromUnicode data is omitted from
* the file and needs to be reconstituted at load time.
* This requires a utf8Friendly format with an additional mbcsIndex table for fast
* (and UTF-8-friendly) fromUnicode conversion for Unicode code points up to maxFastUChar.
* (For details about these structures see below, and see ucnvmbcs.h.)
*
* utf8Friendly also implies that the fromUnicode mappings are stored in ascending order
* of the Unicode code points. (This requires that the .ucm file has the |0 etc.
* precision markers for all mappings.)
*
* All fallbacks have been moved to the extension table, leaving only roundtrips in the
* omitted data that can be reconstituted from the toUnicode data.
*
* Of the stage 2 table, the part corresponding to maxFastUChar and below is omitted.
* With only roundtrip mappings in the base fromUnicode data, this part is fully
* redundant with the mbcsIndex and will be reconstituted from that (also using the
* stage 1 table which contains the information about how stage 2 was compacted).
*
* The rest of the stage 2 table, the part for code points above maxFastUChar,
* is stored in the file and will be appended to the reconstituted part.
*
* The entire fromUBytes array is omitted from the file and will be reconstitued.
* This is done by enumerating all toUnicode roundtrip mappings, performing
* each mapping (using the stage 1 and reconstituted stage 2 tables) and
* writing instead of reading the byte values.
*
* _MBCSHeader version 4.3
*
* Change from version 4.2:
* - Optional utf8Friendly data structures, with 64-entry stage 3 block
* allocation for parts of the BMP, and an additional mbcsIndex in non-SBCS
* files which can be used instead of stages 1 & 2.
* Faster lookups for roundtrips from most commonly used characters,
* and lookups from UTF-8 byte sequences with a natural bit distribution.
* See ucnvmbcs.h for more details.
*
* Change from version 4.1:
* - Added an optional extension table structure at the end of the .cnv file.
* It is present if the upper bits of the header flags field contains a non-zero
* byte offset to it.
* Files that contain only a conversion table and no base table
* use the special outputType MBCS_OUTPUT_EXT_ONLY.
* These contain the base table name between the MBCS header and the extension
* data.
*
* Change from version 4.0:
* - Replace header.reserved with header.fromUBytesLength so that all
* fields in the data have length.
*
* Changes from version 3 (for performance improvements):
* - new bit distribution for state table entries
* - reordered action codes
* - new data structure for single-byte fromUnicode
* + stage 2 only contains indexes
* + stage 3 stores 16 bits per character with classification bits 15..8
* - no multiplier for stage 1 entries
* - stage 2 for non-single-byte codepages contains the index and the flags in
* one 32-bit value
* - 2-byte and 4-byte fromUnicode results are stored directly as 16/32-bit integers
*
* For more details about old versions of the MBCS data structure, see
* the corresponding versions of this file.
*
* Converting stateless codepage data ---------------------------------------***
* (or codepage data with simple states) to Unicode.
*
* Data structure and algorithm for converting from complex legacy codepages
* to Unicode. (Designed before 2000-may-22.)
*
* The basic idea is that the structure of legacy codepages can be described
* with state tables.
* When reading a byte stream, each input byte causes a state transition.
* Some transitions result in the output of a code point, some result in
* "unassigned" or "illegal" output.
* This is used here for character conversion.
*
* The data structure begins with a state table consisting of a row
* per state, with 256 entries (columns) per row for each possible input
* byte value.
* Each entry is 32 bits wide, with two formats distinguished by
* the sign bit (bit 31):
*
* One format for transitional entries (bit 31 not set) for non-final bytes, and
* one format for final entries (bit 31 set).
* Both formats contain the number of the next state in the same bit
* positions.
* State 0 is the initial state.
*
* Most of the time, the offset values of subsequent states are added
* up to a scalar value. This value will eventually be the index of
* the Unicode code point in a table that follows the state table.
* The effect is that the code points for final state table rows
* are contiguous. The code points of final state rows follow each other
* in the order of the references to those final states by previous
* states, etc.
*
* For some terminal states, the offset is itself the output Unicode
* code point (16 bits for a BMP code point or 20 bits for a supplementary
* code point (stored as code point minus 0x10000 so that 20 bits are enough).
* For others, the code point in the Unicode table is stored with either
* one or two code units: one for BMP code points, two for a pair of
* surrogates.
* All code points for a final state entry take up the same number of code
* units, regardless of whether they all actually _use_ the same number
* of code units. This is necessary for simple array access.
*
* An additional feature comes in with what in ICU is called "fallback"
* mappings:
*
* In addition to round-trippable, precise, 1:1 mappings, there are often
* mappings defined between similar, though not the same, characters.
* Typically, such mappings occur only in fromUnicode mapping tables because
* Unicode has a superset repertoire of most other codepages. However, it
* is possible to provide such mappings in the toUnicode tables, too.
* In this case, the fallback mappings are partly integrated into the
* general state tables because the structure of the encoding includes their
* byte sequences.
* For final entries in an initial state, fallback mappings are stored in
* the entry itself like with roundtrip mappings.
* For other final entries, they are stored in the code units table if
* the entry is for a pair of code units.
* For single-unit results in the code units table, there is no space to
* alternatively hold a fallback mapping; in this case, the code unit
* is stored as U+fffe (unassigned), and the fallback mapping needs to
* be looked up by the scalar offset value in a separate table.
*
* "Unassigned" state entries really mean "structurally unassigned",
* i.e., such a byte sequence will never have a mapping result.
*
* The interpretation of the bits in each entry is as follows:
*
* Bit 31 not set, not a terminal entry ("transitional"):
* 30..24 next state
* 23..0 offset delta, to be added up
*
* Bit 31 set, terminal ("final") entry:
* 30..24 next state (regardless of action code)
* 23..20 action code:
* action codes 0 and 1 result in precise-mapping Unicode code points
* 0 valid byte sequence
* 19..16 not used, 0
* 15..0 16-bit Unicode BMP code point
* never U+fffe or U+ffff
* 1 valid byte sequence
* 19..0 20-bit Unicode supplementary code point
* never U+fffe or U+ffff
*
* action codes 2 and 3 result in fallback (unidirectional-mapping) Unicode code points
* 2 valid byte sequence (fallback)
* 19..16 not used, 0
* 15..0 16-bit Unicode BMP code point as fallback result
* 3 valid byte sequence (fallback)
* 19..0 20-bit Unicode supplementary code point as fallback result
*
* action codes 4 and 5 may result in roundtrip/fallback/unassigned/illegal results
* depending on the code units they result in
* 4 valid byte sequence
* 19..9 not used, 0
* 8..0 final offset delta
* pointing to one 16-bit code unit which may be
* fffe unassigned -- look for a fallback for this offset
* ffff illegal
* 5 valid byte sequence
* 19..9 not used, 0
* 8..0 final offset delta
* pointing to two 16-bit code units
* (typically UTF-16 surrogates)
* the result depends on the first code unit as follows:
* 0000..d7ff roundtrip BMP code point (1st alone)
* d800..dbff roundtrip surrogate pair (1st, 2nd)
* dc00..dfff fallback surrogate pair (1st-400, 2nd)
* e000 roundtrip BMP code point (2nd alone)
* e001 fallback BMP code point (2nd alone)
* fffe unassigned
* ffff illegal
* (the final offset deltas are at most 255 * 2,
* times 2 because of storing code unit pairs)
*
* 6 unassigned byte sequence
* 19..16 not used, 0
* 15..0 16-bit Unicode BMP code point U+fffe (new with version 2)
* this does not contain a final offset delta because the main
* purpose of this action code is to save scalar offset values;
* therefore, fallback values cannot be assigned to byte
* sequences that result in this action code
* 7 illegal byte sequence
* 19..16 not used, 0
* 15..0 16-bit Unicode BMP code point U+ffff (new with version 2)
* 8 state change only
* 19..0 not used, 0
* useful for state changes in simple stateful encodings,
* at Shift-In/Shift-Out codes
*
*
* 9..15 reserved for future use
* current implementations will only perform a state change
* and ignore bits 19..0
*
* An encoding with contiguous ranges of unassigned byte sequences, like
* Shift-JIS and especially EUC-TW, can be stored efficiently by having
* at least two states for the trail bytes:
* One trail byte state that results in code points, and one that only
* has "unassigned" and "illegal" terminal states.
*
* Note: partly by accident, this data structure supports simple stateful
* encodings without any additional logic.
* Currently, only simple Shift-In/Shift-Out schemes are handled with
* appropriate state tables (especially EBCDIC_STATEFUL!).
*
* MBCS version 2 added:
* unassigned and illegal action codes have U+fffe and U+ffff
* instead of unused bits; this is useful for _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP()
*
* Converting from Unicode to codepage bytes --------------------------------***
*
* The conversion data structure for fromUnicode is designed for the known
* structure of Unicode. It maps from 21-bit code points (0..0x10ffff) to
* a sequence of 1..4 bytes, in addition to a flag that indicates if there is
* a roundtrip mapping.
*
* The lookup is done with a 3-stage trie, using 11/6/4 bits for stage 1/2/3
* like in the character properties table.
* The beginning of the trie is at offsetFromUTable, the beginning of stage 3
* with the resulting bytes is at offsetFromUBytes.
*
* Beginning with version 4, single-byte codepages have a significantly different
* trie compared to other codepages.
* In all cases, the entry in stage 1 is directly the index of the block of
* 64 entries in stage 2.
*
* Single-byte lookup:
*
* Stage 2 only contains 16-bit indexes directly to the 16-blocks in stage 3.
* Stage 3 contains one 16-bit word per result:
* Bits 15..8 indicate the kind of result:
* f roundtrip result
* c fallback result from private-use code point
* 8 fallback result from other code points
* 0 unassigned
* Bits 7..0 contain the codepage byte. A zero byte is always possible.
*
* In version 4.3, the runtime code can build an sbcsIndex for a utf8Friendly
* file. For 2-byte UTF-8 byte sequences and some 3-byte sequences the lookup
* becomes a 2-stage (single-index) trie lookup with 6 bits for stage 3.
* ASCII code points can be looked up with a linear array access into stage 3.
* See maxFastUChar and other details in ucnvmbcs.h.
*
* Multi-byte lookup:
*
* Stage 2 contains a 32-bit word for each 16-block in stage 3:
* Bits 31..16 contain flags for which stage 3 entries contain roundtrip results
* test: MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c)
* If this test is false, then a non-zero result will be interpreted as
* a fallback mapping.
* Bits 15..0 contain the index to stage 3, which must be multiplied by 16*(bytes per char)
*
* Stage 3 contains 2, 3, or 4 bytes per result.
* 2 or 4 bytes are stored as uint16_t/uint32_t in platform endianness,
* while 3 bytes are stored as bytes in big-endian order.
* Leading zero bytes are ignored, and the number of bytes is counted.
* A zero byte mapping result is possible as a roundtrip result.
* For some output types, the actual result is processed from this;
* see ucnv_MBCSFromUnicodeWithOffsets().
*
* Note that stage 1 always contains 0x440=1088 entries (0x440==0x110000>>10),
* or (version 3 and up) for BMP-only codepages, it contains 64 entries.
*
* In version 4.3, a utf8Friendly file contains an mbcsIndex table.
* For 2-byte UTF-8 byte sequences and most 3-byte sequences the lookup
* becomes a 2-stage (single-index) trie lookup with 6 bits for stage 3.
* ASCII code points can be looked up with a linear array access into stage 3.
* See maxFastUChar, mbcsIndex and other details in ucnvmbcs.h.
*
* In version 3, stage 2 blocks may overlap by multiples of the multiplier
* for compaction.
* In version 4, stage 2 blocks (and for single-byte codepages, stage 3 blocks)
* may overlap by any number of entries.
*
* MBCS version 2 added:
* the converter checks for known output types, which allows
* adding new ones without crashing an unaware converter
*/
static const UConverterImpl _SBCSUTF8Impl;
static const UConverterImpl _DBCSUTF8Impl;
/* GB 18030 data ------------------------------------------------------------ */
/* helper macros for linear values for GB 18030 four-byte sequences */
#define LINEAR_18030(a, b, c, d) ((((a)*10+(b))*126L+(c))*10L+(d))
#define LINEAR_18030_BASE LINEAR_18030(0x81, 0x30, 0x81, 0x30)
#define LINEAR(x) LINEAR_18030(x>>24, (x>>16)&0xff, (x>>8)&0xff, x&0xff)
/*
* Some ranges of GB 18030 where both the Unicode code points and the
* GB four-byte sequences are contiguous and are handled algorithmically by
* the special callback functions below.
* The values are start & end of Unicode & GB codes.
*
* Note that single surrogates are not mapped by GB 18030
* as of the re-released mapping tables from 2000-nov-30.
*/
static const uint32_t
gb18030Ranges[14][4]={
{0x10000, 0x10FFFF, LINEAR(0x90308130), LINEAR(0xE3329A35)},
{0x9FA6, 0xD7FF, LINEAR(0x82358F33), LINEAR(0x8336C738)},
{0x0452, 0x1E3E, LINEAR(0x8130D330), LINEAR(0x8135F436)},
{0x1E40, 0x200F, LINEAR(0x8135F438), LINEAR(0x8136A531)},
{0xE865, 0xF92B, LINEAR(0x8336D030), LINEAR(0x84308534)},
{0x2643, 0x2E80, LINEAR(0x8137A839), LINEAR(0x8138FD38)},
{0xFA2A, 0xFE2F, LINEAR(0x84309C38), LINEAR(0x84318537)},
{0x3CE1, 0x4055, LINEAR(0x8231D438), LINEAR(0x8232AF32)},
{0x361B, 0x3917, LINEAR(0x8230A633), LINEAR(0x8230F237)},
{0x49B8, 0x4C76, LINEAR(0x8234A131), LINEAR(0x8234E733)},
{0x4160, 0x4336, LINEAR(0x8232C937), LINEAR(0x8232F837)},
{0x478E, 0x4946, LINEAR(0x8233E838), LINEAR(0x82349638)},
{0x44D7, 0x464B, LINEAR(0x8233A339), LINEAR(0x8233C931)},
{0xFFE6, 0xFFFF, LINEAR(0x8431A234), LINEAR(0x8431A439)}
};
/* bit flag for UConverter.options indicating GB 18030 special handling */
#define _MBCS_OPTION_GB18030 0x8000
/* bit flag for UConverter.options indicating KEIS,JEF,JIF special handling */
#define _MBCS_OPTION_KEIS 0x01000
#define _MBCS_OPTION_JEF 0x02000
#define _MBCS_OPTION_JIPS 0x04000
#define KEIS_SO_CHAR_1 0x0A
#define KEIS_SO_CHAR_2 0x42
#define KEIS_SI_CHAR_1 0x0A
#define KEIS_SI_CHAR_2 0x41
#define JEF_SO_CHAR 0x28
#define JEF_SI_CHAR 0x29
#define JIPS_SO_CHAR_1 0x1A
#define JIPS_SO_CHAR_2 0x70
#define JIPS_SI_CHAR_1 0x1A
#define JIPS_SI_CHAR_2 0x71
enum SISO_Option {
SI,
SO
};
typedef enum SISO_Option SISO_Option;
static int32_t getSISOBytes(SISO_Option option, uint32_t cnvOption, uint8_t *value) {
int32_t SISOLength = 0;
switch (option) {
case SI:
if ((cnvOption&_MBCS_OPTION_KEIS)!=0) {
value[0] = KEIS_SI_CHAR_1;
value[1] = KEIS_SI_CHAR_2;
SISOLength = 2;
} else if ((cnvOption&_MBCS_OPTION_JEF)!=0) {
value[0] = JEF_SI_CHAR;
SISOLength = 1;
} else if ((cnvOption&_MBCS_OPTION_JIPS)!=0) {
value[0] = JIPS_SI_CHAR_1;
value[1] = JIPS_SI_CHAR_2;
SISOLength = 2;
} else {
value[0] = UCNV_SI;
SISOLength = 1;
}
break;
case SO:
if ((cnvOption&_MBCS_OPTION_KEIS)!=0) {
value[0] = KEIS_SO_CHAR_1;
value[1] = KEIS_SO_CHAR_2;
SISOLength = 2;
} else if ((cnvOption&_MBCS_OPTION_JEF)!=0) {
value[0] = JEF_SO_CHAR;
SISOLength = 1;
} else if ((cnvOption&_MBCS_OPTION_JIPS)!=0) {
value[0] = JIPS_SO_CHAR_1;
value[1] = JIPS_SO_CHAR_2;
SISOLength = 2;
} else {
value[0] = UCNV_SO;
SISOLength = 1;
}
break;
default:
/* Should never happen. */
break;
}
return SISOLength;
}
/* Miscellaneous ------------------------------------------------------------ */
/**
* Callback from ucnv_MBCSEnumToUnicode(), takes 32 mappings from
* consecutive sequences of bytes, starting from the one encoded in value,
* to Unicode code points. (Multiple mappings to reduce per-function call overhead.)
* Does not currently support m:n mappings or reverse fallbacks.
* This function will not be called for sequences of bytes with leading zeros.
*
* @param context an opaque pointer, as passed into ucnv_MBCSEnumToUnicode()
* @param value contains 1..4 bytes of the first byte sequence, right-aligned
* @param codePoints resulting Unicode code points, or negative if a byte sequence does
* not map to anything
* @return TRUE to continue enumeration, FALSE to stop
*/
typedef UBool U_CALLCONV
UConverterEnumToUCallback(const void *context, uint32_t value, UChar32 codePoints[32]);
/* similar to ucnv_MBCSGetNextUChar() but recursive */
static UBool
enumToU(UConverterMBCSTable *mbcsTable, int8_t stateProps[],
int32_t state, uint32_t offset,
uint32_t value,
UConverterEnumToUCallback *callback, const void *context,
UErrorCode *pErrorCode) {
UChar32 codePoints[32];
const int32_t *row;
const uint16_t *unicodeCodeUnits;
UChar32 anyCodePoints;
int32_t b, limit;
row=mbcsTable->stateTable[state];
unicodeCodeUnits=mbcsTable->unicodeCodeUnits;
value<<=8;
anyCodePoints=-1; /* becomes non-negative if there is a mapping */
b=(stateProps[state]&0x38)<<2;
if(b==0 && stateProps[state]>=0x40) {
/* skip byte sequences with leading zeros because they are not stored in the fromUnicode table */
codePoints[0]=U_SENTINEL;
b=1;
}
limit=((stateProps[state]&7)+1)<<5;
while(b<limit) {
int32_t entry=row[b];
if(MBCS_ENTRY_IS_TRANSITION(entry)) {
int32_t nextState=MBCS_ENTRY_TRANSITION_STATE(entry);
if(stateProps[nextState]>=0) {
/* recurse to a state with non-ignorable actions */
if(!enumToU(
mbcsTable, stateProps, nextState,
offset+MBCS_ENTRY_TRANSITION_OFFSET(entry),
value|(uint32_t)b,
callback, context,
pErrorCode)) {
return FALSE;
}
}
codePoints[b&0x1f]=U_SENTINEL;
} else {
UChar32 c;
int32_t action;
/*
* An if-else-if chain provides more reliable performance for
* the most common cases compared to a switch.
*/
action=MBCS_ENTRY_FINAL_ACTION(entry);
if(action==MBCS_STATE_VALID_DIRECT_16) {
/* output BMP code point */
c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
} else if(action==MBCS_STATE_VALID_16) {
int32_t finalOffset=offset+MBCS_ENTRY_FINAL_VALUE_16(entry);
c=unicodeCodeUnits[finalOffset];
if(c<0xfffe) {
/* output BMP code point */
} else {
c=U_SENTINEL;
}
} else if(action==MBCS_STATE_VALID_16_PAIR) {
int32_t finalOffset=offset+MBCS_ENTRY_FINAL_VALUE_16(entry);
c=unicodeCodeUnits[finalOffset++];
if(c<0xd800) {
/* output BMP code point below 0xd800 */
} else if(c<=0xdbff) {
/* output roundtrip or fallback supplementary code point */
c=((c&0x3ff)<<10)+unicodeCodeUnits[finalOffset]+(0x10000-0xdc00);
} else if(c==0xe000) {
/* output roundtrip BMP code point above 0xd800 or fallback BMP code point */
c=unicodeCodeUnits[finalOffset];
} else {
c=U_SENTINEL;
}
} else if(action==MBCS_STATE_VALID_DIRECT_20) {
/* output supplementary code point */
c=(UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000);
} else {
c=U_SENTINEL;
}
codePoints[b&0x1f]=c;
anyCodePoints&=c;
}
if(((++b)&0x1f)==0) {
if(anyCodePoints>=0) {
if(!callback(context, value|(uint32_t)(b-0x20), codePoints)) {
return FALSE;
}
anyCodePoints=-1;
}
}
}
return TRUE;
}
/*
* Only called if stateProps[state]==-1.
* A recursive call may do stateProps[state]|=0x40 if this state is the target of an
* MBCS_STATE_CHANGE_ONLY.
*/
static int8_t
getStateProp(const int32_t (*stateTable)[256], int8_t stateProps[], int state) {
const int32_t *row;
int32_t min, max, entry, nextState;
row=stateTable[state];
stateProps[state]=0;
/* find first non-ignorable state */
for(min=0;; ++min) {
entry=row[min];
nextState=MBCS_ENTRY_STATE(entry);
if(stateProps[nextState]==-1) {
getStateProp(stateTable, stateProps, nextState);
}
if(MBCS_ENTRY_IS_TRANSITION(entry)) {
if(stateProps[nextState]>=0) {
break;
}
} else if(MBCS_ENTRY_FINAL_ACTION(entry)<MBCS_STATE_UNASSIGNED) {
break;
}
if(min==0xff) {
stateProps[state]=-0x40; /* (int8_t)0xc0 */
return stateProps[state];
}
}
stateProps[state]|=(int8_t)((min>>5)<<3);
/* find last non-ignorable state */
for(max=0xff; min<max; --max) {
entry=row[max];
nextState=MBCS_ENTRY_STATE(entry);
if(stateProps[nextState]==-1) {
getStateProp(stateTable, stateProps, nextState);
}
if(MBCS_ENTRY_IS_TRANSITION(entry)) {
if(stateProps[nextState]>=0) {
break;
}
} else if(MBCS_ENTRY_FINAL_ACTION(entry)<MBCS_STATE_UNASSIGNED) {
break;
}
}
stateProps[state]|=(int8_t)(max>>5);
/* recurse further and collect direct-state information */
while(min<=max) {
entry=row[min];
nextState=MBCS_ENTRY_STATE(entry);
if(stateProps[nextState]==-1) {
getStateProp(stateTable, stateProps, nextState);
}
if(MBCS_ENTRY_IS_FINAL(entry)) {
stateProps[nextState]|=0x40;
if(MBCS_ENTRY_FINAL_ACTION(entry)<=MBCS_STATE_FALLBACK_DIRECT_20) {
stateProps[state]|=0x40;
}
}
++min;
}
return stateProps[state];
}
/*
* Internal function enumerating the toUnicode data of an MBCS converter.
* Currently only used for reconstituting data for a MBCS_OPT_NO_FROM_U
* table, but could also be used for a future ucnv_getUnicodeSet() option
* that includes reverse fallbacks (after updating this function's implementation).
* Currently only handles roundtrip mappings.
* Does not currently handle extensions.
*/
static void
ucnv_MBCSEnumToUnicode(UConverterMBCSTable *mbcsTable,
UConverterEnumToUCallback *callback, const void *context,
UErrorCode *pErrorCode) {
/*
* Properties for each state, to speed up the enumeration.
* Ignorable actions are unassigned/illegal/state-change-only:
* They do not lead to mappings.
*
* Bits 7..6:
* 1 direct/initial state (stateful converters have multiple)
* 0 non-initial state with transitions or with non-ignorable result actions
* -1 final state with only ignorable actions
*
* Bits 5..3:
* The lowest byte value with non-ignorable actions is
* value<<5 (rounded down).
*
* Bits 2..0:
* The highest byte value with non-ignorable actions is
* (value<<5)&0x1f (rounded up).
*/
int8_t stateProps[MBCS_MAX_STATE_COUNT];
int32_t state;
uprv_memset(stateProps, -1, sizeof(stateProps));
/* recurse from state 0 and set all stateProps */
getStateProp(mbcsTable->stateTable, stateProps, 0);
for(state=0; state<mbcsTable->countStates; ++state) {
/*if(stateProps[state]==-1) {
printf("unused/unreachable <icu:state> %d\n", state);
}*/
if(stateProps[state]>=0x40) {
/* start from each direct state */
enumToU(
mbcsTable, stateProps, state, 0, 0,
callback, context,
pErrorCode);
}
}
}
U_CFUNC void
ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData,
const USetAdder *sa,
UConverterUnicodeSet which,
UConverterSetFilter filter,
UErrorCode *pErrorCode) {
const UConverterMBCSTable *mbcsTable;
const uint16_t *table;
uint32_t st3;
uint16_t st1, maxStage1, st2;
UChar32 c;
/* enumerate the from-Unicode trie table */
mbcsTable=&sharedData->mbcs;
table=mbcsTable->fromUnicodeTable;
if(mbcsTable->unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
maxStage1=0x440;
} else {
maxStage1=0x40;
}
c=0; /* keep track of the current code point while enumerating */
if(mbcsTable->outputType==MBCS_OUTPUT_1) {
const uint16_t *stage2, *stage3, *results;
uint16_t minValue;
results=(const uint16_t *)mbcsTable->fromUnicodeBytes;
/*
* Set a threshold variable for selecting which mappings to use.
* See ucnv_MBCSSingleFromBMPWithOffsets() and
* MBCS_SINGLE_RESULT_FROM_U() for details.
*/
if(which==UCNV_ROUNDTRIP_SET) {
/* use only roundtrips */
minValue=0xf00;
} else /* UCNV_ROUNDTRIP_AND_FALLBACK_SET */ {
/* use all roundtrip and fallback results */
minValue=0x800;
}
for(st1=0; st1<maxStage1; ++st1) {
st2=table[st1];
if(st2>maxStage1) {
stage2=table+st2;
for(st2=0; st2<64; ++st2) {
if((st3=stage2[st2])!=0) {
/* read the stage 3 block */
stage3=results+st3;
do {
if(*stage3++>=minValue) {
sa->add(sa->set, c);
}
} while((++c&0xf)!=0);
} else {
c+=16; /* empty stage 3 block */
}
}
} else {
c+=1024; /* empty stage 2 block */
}
}
} else {
const uint32_t *stage2;
const uint8_t *stage3, *bytes;
uint32_t st3Multiplier;
uint32_t value;
UBool useFallback;
bytes=mbcsTable->fromUnicodeBytes;
useFallback=(UBool)(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET);
switch(mbcsTable->outputType) {
case MBCS_OUTPUT_3:
case MBCS_OUTPUT_4_EUC:
st3Multiplier=3;
break;
case MBCS_OUTPUT_4:
st3Multiplier=4;
break;
default:
st3Multiplier=2;
break;
}
for(st1=0; st1<maxStage1; ++st1) {
st2=table[st1];
if(st2>(maxStage1>>1)) {
stage2=(const uint32_t *)table+st2;
for(st2=0; st2<64; ++st2) {
if((st3=stage2[st2])!=0) {
/* read the stage 3 block */
stage3=bytes+st3Multiplier*16*(uint32_t)(uint16_t)st3;
/* get the roundtrip flags for the stage 3 block */
st3>>=16;
/*
* Add code points for which the roundtrip flag is set,
* or which map to non-zero bytes if we use fallbacks.
* See ucnv_MBCSFromUnicodeWithOffsets() for details.
*/
switch(filter) {
case UCNV_SET_FILTER_NONE:
do {
if(st3&1) {
sa->add(sa->set, c);
stage3+=st3Multiplier;
} else if(useFallback) {
uint8_t b=0;
switch(st3Multiplier) {
case 4:
b|=*stage3++;
case 3: /*fall through*/
b|=*stage3++;
case 2: /*fall through*/
b|=stage3[0]|stage3[1];
stage3+=2;
default:
break;
}
if(b!=0) {
sa->add(sa->set, c);
}
}
st3>>=1;
} while((++c&0xf)!=0);
break;
case UCNV_SET_FILTER_DBCS_ONLY:
/* Ignore single-byte results (<0x100). */
do {
if(((st3&1)!=0 || useFallback) && *((const uint16_t *)stage3)>=0x100) {
sa->add(sa->set, c);
}
st3>>=1;
stage3+=2; /* +=st3Multiplier */
} while((++c&0xf)!=0);
break;
case UCNV_SET_FILTER_2022_CN:
/* Only add code points that map to CNS 11643 planes 1 & 2 for non-EXT ISO-2022-CN. */
do {
if(((st3&1)!=0 || useFallback) && ((value=*stage3)==0x81 || value==0x82)) {
sa->add(sa->set, c);
}
st3>>=1;
stage3+=3; /* +=st3Multiplier */
} while((++c&0xf)!=0);
break;
case UCNV_SET_FILTER_SJIS:
/* Only add code points that map to Shift-JIS codes corresponding to JIS X 0208. */
do {
if(((st3&1)!=0 || useFallback) && (value=*((const uint16_t *)stage3))>=0x8140 && value<=0xeffc) {
sa->add(sa->set, c);
}
st3>>=1;
stage3+=2; /* +=st3Multiplier */
} while((++c&0xf)!=0);
break;
case UCNV_SET_FILTER_GR94DBCS:
/* Only add code points that map to ISO 2022 GR 94 DBCS codes (each byte A1..FE). */
do {
if( ((st3&1)!=0 || useFallback) &&
(uint16_t)((value=*((const uint16_t *)stage3)) - 0xa1a1)<=(0xfefe - 0xa1a1) &&
(uint8_t)(value-0xa1)<=(0xfe - 0xa1)
) {
sa->add(sa->set, c);
}
st3>>=1;
stage3+=2; /* +=st3Multiplier */
} while((++c&0xf)!=0);
break;
case UCNV_SET_FILTER_HZ:
/* Only add code points that are suitable for HZ DBCS (lead byte A1..FD). */
do {
if( ((st3&1)!=0 || useFallback) &&
(uint16_t)((value=*((const uint16_t *)stage3))-0xa1a1)<=(0xfdfe - 0xa1a1) &&
(uint8_t)(value-0xa1)<=(0xfe - 0xa1)
) {
sa->add(sa->set, c);
}
st3>>=1;
stage3+=2; /* +=st3Multiplier */
} while((++c&0xf)!=0);
break;
default:
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
return;
}
} else {
c+=16; /* empty stage 3 block */
}
}
} else {
c+=1024; /* empty stage 2 block */
}
}
}
ucnv_extGetUnicodeSet(sharedData, sa, which, filter, pErrorCode);
}
U_CFUNC void
ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData,
const USetAdder *sa,
UConverterUnicodeSet which,
UErrorCode *pErrorCode) {
ucnv_MBCSGetFilteredUnicodeSetForUnicode(
sharedData, sa, which,
sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ?
UCNV_SET_FILTER_DBCS_ONLY :
UCNV_SET_FILTER_NONE,
pErrorCode);
}
static void
ucnv_MBCSGetUnicodeSet(const UConverter *cnv,
const USetAdder *sa,
UConverterUnicodeSet which,
UErrorCode *pErrorCode) {
if(cnv->options&_MBCS_OPTION_GB18030) {
sa->addRange(sa->set, 0, 0xd7ff);
sa->addRange(sa->set, 0xe000, 0x10ffff);
} else {
ucnv_MBCSGetUnicodeSetForUnicode(cnv->sharedData, sa, which, pErrorCode);
}
}
/* conversion extensions for input not in the main table -------------------- */
/*
* Hardcoded extension handling for GB 18030.
* Definition of LINEAR macros and gb18030Ranges see near the beginning of the file.
*
* In the future, conversion extensions may handle m:n mappings and delta tables,
* see http://source.icu-project.org/repos/icu/icuhtml/trunk/design/conversion/conversion_extensions.html
*
* If an input character cannot be mapped, then these functions set an error
* code. The framework will then call the callback function.
*/
/*
* @return if(U_FAILURE) return the code point for cnv->fromUChar32
* else return 0 after output has been written to the target
*/
static UChar32
_extFromU(UConverter *cnv, const UConverterSharedData *sharedData,
UChar32 cp,
const UChar **source, const UChar *sourceLimit,
uint8_t **target, const uint8_t *targetLimit,
int32_t **offsets, int32_t sourceIndex,
UBool flush,
UErrorCode *pErrorCode) {
const int32_t *cx;
cnv->useSubChar1=FALSE;
if( (cx=sharedData->mbcs.extIndexes)!=NULL &&
ucnv_extInitialMatchFromU(
cnv, cx,
cp, source, sourceLimit,
(char **)target, (char *)targetLimit,
offsets, sourceIndex,
flush,
pErrorCode)
) {
return 0; /* an extension mapping handled the input */
}
/* GB 18030 */
if((cnv->options&_MBCS_OPTION_GB18030)!=0) {
const uint32_t *range;
int32_t i;
range=gb18030Ranges[0];
for(i=0; i<sizeof(gb18030Ranges)/sizeof(gb18030Ranges[0]); range+=4, ++i) {
if(range[0]<=(uint32_t)cp && (uint32_t)cp<=range[1]) {
/* found the Unicode code point, output the four-byte sequence for it */
uint32_t linear;
char bytes[4];
/* get the linear value of the first GB 18030 code in this range */
linear=range[2]-LINEAR_18030_BASE;
/* add the offset from the beginning of the range */
linear+=((uint32_t)cp-range[0]);
/* turn this into a four-byte sequence */
bytes[3]=(char)(0x30+linear%10); linear/=10;
bytes[2]=(char)(0x81+linear%126); linear/=126;
bytes[1]=(char)(0x30+linear%10); linear/=10;
bytes[0]=(char)(0x81+linear);
/* output this sequence */
ucnv_fromUWriteBytes(cnv,
bytes, 4, (char **)target, (char *)targetLimit,
offsets, sourceIndex, pErrorCode);
return 0;
}
}
}
/* no mapping */
*pErrorCode=U_INVALID_CHAR_FOUND;
return cp;
}
/*
* Input sequence: cnv->toUBytes[0..length[
* @return if(U_FAILURE) return the length (toULength, byteIndex) for the input
* else return 0 after output has been written to the target
*/
static int8_t
_extToU(UConverter *cnv, const UConverterSharedData *sharedData,
int8_t length,
const uint8_t **source, const uint8_t *sourceLimit,
UChar **target, const UChar *targetLimit,
int32_t **offsets, int32_t sourceIndex,
UBool flush,
UErrorCode *pErrorCode) {
const int32_t *cx;
if( (cx=sharedData->mbcs.extIndexes)!=NULL &&
ucnv_extInitialMatchToU(
cnv, cx,
length, (const char **)source, (const char *)sourceLimit,
target, targetLimit,
offsets, sourceIndex,
flush,
pErrorCode)
) {
return 0; /* an extension mapping handled the input */
}
/* GB 18030 */
if(length==4 && (cnv->options&_MBCS_OPTION_GB18030)!=0) {
const uint32_t *range;
uint32_t linear;
int32_t i;
linear=LINEAR_18030(cnv->toUBytes[0], cnv->toUBytes[1], cnv->toUBytes[2], cnv->toUBytes[3]);
range=gb18030Ranges[0];
for(i=0; i<sizeof(gb18030Ranges)/sizeof(gb18030Ranges[0]); range+=4, ++i) {
if(range[2]<=linear && linear<=range[3]) {
/* found the sequence, output the Unicode code point for it */
*pErrorCode=U_ZERO_ERROR;
/* add the linear difference between the input and start sequences to the start code point */
linear=range[0]+(linear-range[2]);
/* output this code point */
ucnv_toUWriteCodePoint(cnv, linear, target, targetLimit, offsets, sourceIndex, pErrorCode);
return 0;
}
}
}
/* no mapping */
*pErrorCode=U_INVALID_CHAR_FOUND;
return length;
}
/* EBCDIC swap LF<->NL ------------------------------------------------------ */
/*
* This code modifies a standard EBCDIC<->Unicode mapping table for
* OS/390 (z/OS) Unix System Services (Open Edition).
* The difference is in the mapping of Line Feed and New Line control codes:
* Standard EBCDIC maps
*
* <U000A> \x25 |0
* <U0085> \x15 |0
*
* but OS/390 USS EBCDIC swaps the control codes for LF and NL,
* mapping
*
* <U000A> \x15 |0
* <U0085> \x25 |0
*
* This code modifies a loaded standard EBCDIC<->Unicode mapping table
* by copying it into allocated memory and swapping the LF and NL values.
* It allows to support the same EBCDIC charset in both versions without
* duplicating the entire installed table.
*/
/* standard EBCDIC codes */
#define EBCDIC_LF 0x25
#define EBCDIC_NL 0x15
/* standard EBCDIC codes with roundtrip flag as stored in Unicode-to-single-byte tables */
#define EBCDIC_RT_LF 0xf25
#define EBCDIC_RT_NL 0xf15
/* Unicode code points */
#define U_LF 0x0a
#define U_NL 0x85
static UBool
_EBCDICSwapLFNL(UConverterSharedData *sharedData, UErrorCode *pErrorCode) {
UConverterMBCSTable *mbcsTable;
const uint16_t *table, *results;
const uint8_t *bytes;
int32_t (*newStateTable)[256];
uint16_t *newResults;
uint8_t *p;
char *name;
uint32_t stage2Entry;
uint32_t size, sizeofFromUBytes;
mbcsTable=&sharedData->mbcs;
table=mbcsTable->fromUnicodeTable;
bytes=mbcsTable->fromUnicodeBytes;
results=(const uint16_t *)bytes;
/*
* Check that this is an EBCDIC table with SBCS portion -
* SBCS or EBCDIC_STATEFUL with standard EBCDIC LF and NL mappings.
*
* If not, ignore the option. Options are always ignored if they do not apply.
*/
if(!(
(mbcsTable->outputType==MBCS_OUTPUT_1 || mbcsTable->outputType==MBCS_OUTPUT_2_SISO) &&
mbcsTable->stateTable[0][EBCDIC_LF]==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_LF) &&
mbcsTable->stateTable[0][EBCDIC_NL]==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_NL)
)) {
return FALSE;
}
if(mbcsTable->outputType==MBCS_OUTPUT_1) {
if(!(
EBCDIC_RT_LF==MBCS_SINGLE_RESULT_FROM_U(table, results, U_LF) &&
EBCDIC_RT_NL==MBCS_SINGLE_RESULT_FROM_U(table, results, U_NL)
)) {
return FALSE;
}
} else /* MBCS_OUTPUT_2_SISO */ {
stage2Entry=MBCS_STAGE_2_FROM_U(table, U_LF);
if(!(
MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, U_LF)!=0 &&
EBCDIC_LF==MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, U_LF)
)) {
return FALSE;
}
stage2Entry=MBCS_STAGE_2_FROM_U(table, U_NL);
if(!(
MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, U_NL)!=0 &&
EBCDIC_NL==MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, U_NL)
)) {
return FALSE;
}
}
if(mbcsTable->fromUBytesLength>0) {
/*
* We _know_ the number of bytes in the fromUnicodeBytes array
* starting with header.version 4.1.
*/
sizeofFromUBytes=mbcsTable->fromUBytesLength;
} else {
/*
* Otherwise:
* There used to be code to enumerate the fromUnicode
* trie and find the highest entry, but it was removed in ICU 3.2
* because it was not tested and caused a low code coverage number.
* See Jitterbug 3674.
* This affects only some .cnv file formats with a header.version
* below 4.1, and only when swaplfnl is requested.
*
* ucnvmbcs.c revision 1.99 is the last one with the
* ucnv_MBCSSizeofFromUBytes() function.
*/
*pErrorCode=U_INVALID_FORMAT_ERROR;
return FALSE;
}
/*
* The table has an appropriate format.
* Allocate and build
* - a modified to-Unicode state table
* - a modified from-Unicode output array
* - a converter name string with the swap option appended
*/
size=
mbcsTable->countStates*1024+
sizeofFromUBytes+
UCNV_MAX_CONVERTER_NAME_LENGTH+20;
p=(uint8_t *)uprv_malloc(size);
if(p==NULL) {
*pErrorCode=U_MEMORY_ALLOCATION_ERROR;
return FALSE;
}
/* copy and modify the to-Unicode state table */
newStateTable=(int32_t (*)[256])p;
uprv_memcpy(newStateTable, mbcsTable->stateTable, mbcsTable->countStates*1024);
newStateTable[0][EBCDIC_LF]=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_NL);
newStateTable[0][EBCDIC_NL]=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_LF);
/* copy and modify the from-Unicode result table */
newResults=(uint16_t *)newStateTable[mbcsTable->countStates];
uprv_memcpy(newResults, bytes, sizeofFromUBytes);
/* conveniently, the table access macros work on the left side of expressions */
if(mbcsTable->outputType==MBCS_OUTPUT_1) {
MBCS_SINGLE_RESULT_FROM_U(table, newResults, U_LF)=EBCDIC_RT_NL;
MBCS_SINGLE_RESULT_FROM_U(table, newResults, U_NL)=EBCDIC_RT_LF;
} else /* MBCS_OUTPUT_2_SISO */ {
stage2Entry=MBCS_STAGE_2_FROM_U(table, U_LF);
MBCS_VALUE_2_FROM_STAGE_2(newResults, stage2Entry, U_LF)=EBCDIC_NL;
stage2Entry=MBCS_STAGE_2_FROM_U(table, U_NL);
MBCS_VALUE_2_FROM_STAGE_2(newResults, stage2Entry, U_NL)=EBCDIC_LF;
}
/* set the canonical converter name */
name=(char *)newResults+sizeofFromUBytes;
uprv_strcpy(name, sharedData->staticData->name);
uprv_strcat(name, UCNV_SWAP_LFNL_OPTION_STRING);
/* set the pointers */
umtx_lock(NULL);
if(mbcsTable->swapLFNLStateTable==NULL) {
mbcsTable->swapLFNLStateTable=newStateTable;
mbcsTable->swapLFNLFromUnicodeBytes=(uint8_t *)newResults;
mbcsTable->swapLFNLName=name;
newStateTable=NULL;
}
umtx_unlock(NULL);
/* release the allocated memory if another thread beat us to it */
if(newStateTable!=NULL) {
uprv_free(newStateTable);
}
return TRUE;
}
/* reconstitute omitted fromUnicode data ------------------------------------ */
/* for details, compare with genmbcs.c MBCSAddFromUnicode() and transformEUC() */
static UBool U_CALLCONV
writeStage3Roundtrip(const void *context, uint32_t value, UChar32 codePoints[32]) {
UConverterMBCSTable *mbcsTable=(UConverterMBCSTable *)context;
const uint16_t *table;
uint32_t *stage2;
uint8_t *bytes, *p;
UChar32 c;
int32_t i, st3;
table=mbcsTable->fromUnicodeTable;
bytes=(uint8_t *)mbcsTable->fromUnicodeBytes;
/* for EUC outputTypes, modify the value like genmbcs.c's transformEUC() */
switch(mbcsTable->outputType) {
case MBCS_OUTPUT_3_EUC:
if(value<=0xffff) {
/* short sequences are stored directly */
/* code set 0 or 1 */
} else if(value<=0x8effff) {
/* code set 2 */
value&=0x7fff;
} else /* first byte is 0x8f */ {
/* code set 3 */
value&=0xff7f;
}
break;
case MBCS_OUTPUT_4_EUC:
if(value<=0xffffff) {
/* short sequences are stored directly */
/* code set 0 or 1 */
} else if(value<=0x8effffff) {
/* code set 2 */
value&=0x7fffff;
} else /* first byte is 0x8f */ {
/* code set 3 */
value&=0xff7fff;
}
break;
default:
break;
}
for(i=0; i<=0x1f; ++value, ++i) {
c=codePoints[i];
if(c<0) {
continue;
}
/* locate the stage 2 & 3 data */
stage2=((uint32_t *)table)+table[c>>10]+((c>>4)&0x3f);
p=bytes;
st3=(int32_t)(uint16_t)*stage2*16+(c&0xf);
/* write the codepage bytes into stage 3 */
switch(mbcsTable->outputType) {
case MBCS_OUTPUT_3:
case MBCS_OUTPUT_4_EUC:
p+=st3*3;
p[0]=(uint8_t)(value>>16);
p[1]=(uint8_t)(value>>8);
p[2]=(uint8_t)value;
break;
case MBCS_OUTPUT_4:
((uint32_t *)p)[st3]=value;
break;
default:
/* 2 bytes per character */
((uint16_t *)p)[st3]=(uint16_t)value;
break;
}
/* set the roundtrip flag */
*stage2|=(1UL<<(16+(c&0xf)));
}
return TRUE;
}
static void
reconstituteData(UConverterMBCSTable *mbcsTable,
uint32_t stage1Length, uint32_t stage2Length,
uint32_t fullStage2Length, /* lengths are numbers of units, not bytes */
UErrorCode *pErrorCode) {
uint16_t *stage1;
uint32_t *stage2;
uint32_t dataLength=stage1Length*2+fullStage2Length*4+mbcsTable->fromUBytesLength;
mbcsTable->reconstitutedData=(uint8_t *)uprv_malloc(dataLength);
if(mbcsTable->reconstitutedData==NULL) {
*pErrorCode=U_MEMORY_ALLOCATION_ERROR;
return;
}
uprv_memset(mbcsTable->reconstitutedData, 0, dataLength);
/* copy existing data and reroute the pointers */
stage1=(uint16_t *)mbcsTable->reconstitutedData;
uprv_memcpy(stage1, mbcsTable->fromUnicodeTable, stage1Length*2);
stage2=(uint32_t *)(stage1+stage1Length);
uprv_memcpy(stage2+(fullStage2Length-stage2Length),
mbcsTable->fromUnicodeTable+stage1Length,
stage2Length*4);
mbcsTable->fromUnicodeTable=stage1;
mbcsTable->fromUnicodeBytes=(uint8_t *)(stage2+fullStage2Length);
/* indexes into stage 2 count from the bottom of the fromUnicodeTable */
stage2=(uint32_t *)stage1;
/* reconstitute the initial part of stage 2 from the mbcsIndex */
{
int32_t stageUTF8Length=((int32_t)mbcsTable->maxFastUChar+1)>>6;
int32_t stageUTF8Index=0;
int32_t st1, st2, st3, i;
for(st1=0; stageUTF8Index<stageUTF8Length; ++st1) {
st2=stage1[st1];
if(st2!=stage1Length/2) {
/* each stage 2 block has 64 entries corresponding to 16 entries in the mbcsIndex */
for(i=0; i<16; ++i) {
st3=mbcsTable->mbcsIndex[stageUTF8Index++];
if(st3!=0) {
/* an stage 2 entry's index is per stage 3 16-block, not per stage 3 entry */
st3>>=4;
/*
* 4 stage 2 entries point to 4 consecutive stage 3 16-blocks which are
* allocated together as a single 64-block for access from the mbcsIndex
*/
stage2[st2++]=st3++;
stage2[st2++]=st3++;
stage2[st2++]=st3++;
stage2[st2++]=st3;
} else {
/* no stage 3 block, skip */
st2+=4;
}
}
} else {
/* no stage 2 block, skip */
stageUTF8Index+=16;
}
}
}
/* reconstitute fromUnicodeBytes with roundtrips from toUnicode data */
ucnv_MBCSEnumToUnicode(mbcsTable, writeStage3Roundtrip, mbcsTable, pErrorCode);
}
/* MBCS setup functions ----------------------------------------------------- */
static void
ucnv_MBCSLoad(UConverterSharedData *sharedData,
UConverterLoadArgs *pArgs,
const uint8_t *raw,
UErrorCode *pErrorCode) {
UDataInfo info;
UConverterMBCSTable *mbcsTable=&sharedData->mbcs;
_MBCSHeader *header=(_MBCSHeader *)raw;
uint32_t offset;
uint32_t headerLength;
UBool noFromU=FALSE;
if(header->version[0]==4) {
headerLength=MBCS_HEADER_V4_LENGTH;
} else if(header->version[0]==5 && header->version[1]>=3 &&
(header->options&MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK)==0) {
headerLength=header->options&MBCS_OPT_LENGTH_MASK;
noFromU=(UBool)((header->options&MBCS_OPT_NO_FROM_U)!=0);
} else {
*pErrorCode=U_INVALID_TABLE_FORMAT;
return;
}
mbcsTable->outputType=(uint8_t)header->flags;
if(noFromU && mbcsTable->outputType==MBCS_OUTPUT_1) {
*pErrorCode=U_INVALID_TABLE_FORMAT;
return;
}
/* extension data, header version 4.2 and higher */
offset=header->flags>>8;
if(offset!=0) {
mbcsTable->extIndexes=(const int32_t *)(raw+offset);
}
if(mbcsTable->outputType==MBCS_OUTPUT_EXT_ONLY) {
UConverterLoadArgs args={ 0 };
UConverterSharedData *baseSharedData;
const int32_t *extIndexes;
const char *baseName;
/* extension-only file, load the base table and set values appropriately */
if((extIndexes=mbcsTable->extIndexes)==NULL) {
/* extension-only file without extension */
*pErrorCode=U_INVALID_TABLE_FORMAT;
return;
}
if(pArgs->nestedLoads!=1) {
/* an extension table must not be loaded as a base table */
*pErrorCode=U_INVALID_TABLE_FILE;
return;
}
/* load the base table */
baseName=(const char *)header+headerLength*4;
if(0==uprv_strcmp(baseName, sharedData->staticData->name)) {
/* forbid loading this same extension-only file */
*pErrorCode=U_INVALID_TABLE_FORMAT;
return;
}
/* TODO parse package name out of the prefix of the base name in the extension .cnv file? */
args.size=sizeof(UConverterLoadArgs);
args.nestedLoads=2;
args.onlyTestIsLoadable=pArgs->onlyTestIsLoadable;
args.reserved=pArgs->reserved;
args.options=pArgs->options;
args.pkg=pArgs->pkg;
args.name=baseName;
baseSharedData=ucnv_load(&args, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
return;
}
if( baseSharedData->staticData->conversionType!=UCNV_MBCS ||
baseSharedData->mbcs.baseSharedData!=NULL
) {
ucnv_unload(baseSharedData);
*pErrorCode=U_INVALID_TABLE_FORMAT;
return;
}
if(pArgs->onlyTestIsLoadable) {
/*
* Exit as soon as we know that we can load the converter
* and the format is valid and supported.
* The worst that can happen in the following code is a memory
* allocation error.
*/
ucnv_unload(baseSharedData);
return;
}
/* copy the base table data */
uprv_memcpy(mbcsTable, &baseSharedData->mbcs, sizeof(UConverterMBCSTable));
/* overwrite values with relevant ones for the extension converter */
mbcsTable->baseSharedData=baseSharedData;
mbcsTable->extIndexes=extIndexes;
/*
* It would be possible to share the swapLFNL data with a base converter,
* but the generated name would have to be different, and the memory
* would have to be free'd only once.
* It is easier to just create the data for the extension converter
* separately when it is requested.
*/
mbcsTable->swapLFNLStateTable=NULL;
mbcsTable->swapLFNLFromUnicodeBytes=NULL;
mbcsTable->swapLFNLName=NULL;
/*
* The reconstitutedData must be deleted only when the base converter
* is unloaded.
*/
mbcsTable->reconstitutedData=NULL;
/*
* Set a special, runtime-only outputType if the extension converter
* is a DBCS version of a base converter that also maps single bytes.
*/
if( sharedData->staticData->conversionType==UCNV_DBCS ||
(sharedData->staticData->conversionType==UCNV_MBCS &&
sharedData->staticData->minBytesPerChar>=2)
) {
if(baseSharedData->mbcs.outputType==MBCS_OUTPUT_2_SISO) {
/* the base converter is SI/SO-stateful */
int32_t entry;
/* get the dbcs state from the state table entry for SO=0x0e */
entry=mbcsTable->stateTable[0][0xe];
if( MBCS_ENTRY_IS_FINAL(entry) &&
MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_CHANGE_ONLY &&
MBCS_ENTRY_FINAL_STATE(entry)!=0
) {
mbcsTable->dbcsOnlyState=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry);
mbcsTable->outputType=MBCS_OUTPUT_DBCS_ONLY;
}
} else if(
baseSharedData->staticData->conversionType==UCNV_MBCS &&
baseSharedData->staticData->minBytesPerChar==1 &&
baseSharedData->staticData->maxBytesPerChar==2 &&
mbcsTable->countStates<=127
) {
/* non-stateful base converter, need to modify the state table */
int32_t (*newStateTable)[256];
int32_t *state;
int32_t i, count;
/* allocate a new state table and copy the base state table contents */
count=mbcsTable->countStates;
newStateTable=(int32_t (*)[256])uprv_malloc((count+1)*1024);
if(newStateTable==NULL) {
ucnv_unload(baseSharedData);
*pErrorCode=U_MEMORY_ALLOCATION_ERROR;
return;
}
uprv_memcpy(newStateTable, mbcsTable->stateTable, count*1024);
/* change all final single-byte entries to go to a new all-illegal state */
state=newStateTable[0];
for(i=0; i<256; ++i) {
if(MBCS_ENTRY_IS_FINAL(state[i])) {
state[i]=MBCS_ENTRY_TRANSITION(count, 0);
}
}
/* build the new all-illegal state */
state=newStateTable[count];
for(i=0; i<256; ++i) {
state[i]=MBCS_ENTRY_FINAL(0, MBCS_STATE_ILLEGAL, 0);
}
mbcsTable->stateTable=(const int32_t (*)[256])newStateTable;
mbcsTable->countStates=(uint8_t)(count+1);
mbcsTable->stateTableOwned=TRUE;
mbcsTable->outputType=MBCS_OUTPUT_DBCS_ONLY;
}
}
/*
* unlike below for files with base tables, do not get the unicodeMask
* from the sharedData; instead, use the base table's unicodeMask,
* which we copied in the memcpy above;
* this is necessary because the static data unicodeMask, especially
* the UCNV_HAS_SUPPLEMENTARY flag, is part of the base table data
*/
} else {
/* conversion file with a base table; an additional extension table is optional */
/* make sure that the output type is known */
switch(mbcsTable->outputType) {
case MBCS_OUTPUT_1:
case MBCS_OUTPUT_2:
case MBCS_OUTPUT_3:
case MBCS_OUTPUT_4:
case MBCS_OUTPUT_3_EUC:
case MBCS_OUTPUT_4_EUC:
case MBCS_OUTPUT_2_SISO:
/* OK */
break;
default:
*pErrorCode=U_INVALID_TABLE_FORMAT;
return;
}
if(pArgs->onlyTestIsLoadable) {
/*
* Exit as soon as we know that we can load the converter
* and the format is valid and supported.
* The worst that can happen in the following code is a memory
* allocation error.
*/
return;
}
mbcsTable->countStates=(uint8_t)header->countStates;
mbcsTable->countToUFallbacks=header->countToUFallbacks;
mbcsTable->stateTable=(const int32_t (*)[256])(raw+headerLength*4);
mbcsTable->toUFallbacks=(const _MBCSToUFallback *)(mbcsTable->stateTable+header->countStates);
mbcsTable->unicodeCodeUnits=(const uint16_t *)(raw+header->offsetToUCodeUnits);
mbcsTable->fromUnicodeTable=(const uint16_t *)(raw+header->offsetFromUTable);
mbcsTable->fromUnicodeBytes=(const uint8_t *)(raw+header->offsetFromUBytes);
mbcsTable->fromUBytesLength=header->fromUBytesLength;
/*
* converter versions 6.1 and up contain a unicodeMask that is
* used here to select the most efficient function implementations
*/
info.size=sizeof(UDataInfo);
udata_getInfo((UDataMemory *)sharedData->dataMemory, &info);
if(info.formatVersion[0]>6 || (info.formatVersion[0]==6 && info.formatVersion[1]>=1)) {
/* mask off possible future extensions to be safe */
mbcsTable->unicodeMask=(uint8_t)(sharedData->staticData->unicodeMask&3);
} else {
/* for older versions, assume worst case: contains anything possible (prevent over-optimizations) */
mbcsTable->unicodeMask=UCNV_HAS_SUPPLEMENTARY|UCNV_HAS_SURROGATES;
}
/*
* _MBCSHeader.version 4.3 adds utf8Friendly data structures.
* Check for the header version, SBCS vs. MBCS, and for whether the
* data structures are optimized for code points as high as what the
* runtime code is designed for.
* The implementation does not handle mapping tables with entries for
* unpaired surrogates.
*/
if( header->version[1]>=3 &&
(mbcsTable->unicodeMask&UCNV_HAS_SURROGATES)==0 &&
(mbcsTable->countStates==1 ?
(header->version[2]>=(SBCS_FAST_MAX>>8)) :
(header->version[2]>=(MBCS_FAST_MAX>>8))
)
) {
mbcsTable->utf8Friendly=TRUE;
if(mbcsTable->countStates==1) {
/*
* SBCS: Stage 3 is allocated in 64-entry blocks for U+0000..SBCS_FAST_MAX or higher.
* Build a table with indexes to each block, to be used instead of
* the regular stage 1/2 table.
*/
int32_t i;
for(i=0; i<(SBCS_FAST_LIMIT>>6); ++i) {
mbcsTable->sbcsIndex[i]=mbcsTable->fromUnicodeTable[mbcsTable->fromUnicodeTable[i>>4]+((i<<2)&0x3c)];
}
/* set SBCS_FAST_MAX to reflect the reach of sbcsIndex[] even if header->version[2]>(SBCS_FAST_MAX>>8) */
mbcsTable->maxFastUChar=SBCS_FAST_MAX;
} else {
/*
* MBCS: Stage 3 is allocated in 64-entry blocks for U+0000..MBCS_FAST_MAX or higher.
* The .cnv file is prebuilt with an additional stage table with indexes
* to each block.
*/
mbcsTable->mbcsIndex=(const uint16_t *)
(mbcsTable->fromUnicodeBytes+
(noFromU ? 0 : mbcsTable->fromUBytesLength));
mbcsTable->maxFastUChar=(((UChar)header->version[2])<<8)|0xff;
}
}
/* calculate a bit set of 4 ASCII characters per bit that round-trip to ASCII bytes */
{
uint32_t asciiRoundtrips=0xffffffff;
int32_t i;
for(i=0; i<0x80; ++i) {
if(mbcsTable->stateTable[0][i]!=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, i)) {
asciiRoundtrips&=~((uint32_t)1<<(i>>2));
}
}
mbcsTable->asciiRoundtrips=asciiRoundtrips;
}
if(noFromU) {
uint32_t stage1Length=
mbcsTable->unicodeMask&UCNV_HAS_SUPPLEMENTARY ?
0x440 : 0x40;
uint32_t stage2Length=
(header->offsetFromUBytes-header->offsetFromUTable)/4-
stage1Length/2;
reconstituteData(mbcsTable, stage1Length, stage2Length, header->fullStage2Length, pErrorCode);
}
}
/* Set the impl pointer here so that it is set for both extension-only and base tables. */
if(mbcsTable->utf8Friendly) {
if(mbcsTable->countStates==1) {
sharedData->impl=&_SBCSUTF8Impl;
} else {
if(mbcsTable->outputType==MBCS_OUTPUT_2) {
sharedData->impl=&_DBCSUTF8Impl;
}
}
}
if(mbcsTable->outputType==MBCS_OUTPUT_DBCS_ONLY || mbcsTable->outputType==MBCS_OUTPUT_2_SISO) {
/*
* MBCS_OUTPUT_DBCS_ONLY: No SBCS mappings, therefore ASCII does not roundtrip.
* MBCS_OUTPUT_2_SISO: Bypass the ASCII fastpath to handle prevLength correctly.
*/
mbcsTable->asciiRoundtrips=0;
}
}
static void
ucnv_MBCSUnload(UConverterSharedData *sharedData) {
UConverterMBCSTable *mbcsTable=&sharedData->mbcs;
if(mbcsTable->swapLFNLStateTable!=NULL) {
uprv_free(mbcsTable->swapLFNLStateTable);
}
if(mbcsTable->stateTableOwned) {
uprv_free((void *)mbcsTable->stateTable);
}
if(mbcsTable->baseSharedData!=NULL) {
ucnv_unload(mbcsTable->baseSharedData);
}
if(mbcsTable->reconstitutedData!=NULL) {
uprv_free(mbcsTable->reconstitutedData);
}
}
static void
ucnv_MBCSOpen(UConverter *cnv,
UConverterLoadArgs *pArgs,
UErrorCode *pErrorCode) {
UConverterMBCSTable *mbcsTable;
const int32_t *extIndexes;
uint8_t outputType;
int8_t maxBytesPerUChar;
if(pArgs->onlyTestIsLoadable) {
return;
}
mbcsTable=&cnv->sharedData->mbcs;
outputType=mbcsTable->outputType;
if(outputType==MBCS_OUTPUT_DBCS_ONLY) {
/* the swaplfnl option does not apply, remove it */
cnv->options=pArgs->options&=~UCNV_OPTION_SWAP_LFNL;
}
if((pArgs->options&UCNV_OPTION_SWAP_LFNL)!=0) {
/* do this because double-checked locking is broken */
UBool isCached;
umtx_lock(NULL);
isCached=mbcsTable->swapLFNLStateTable!=NULL;
umtx_unlock(NULL);
if(!isCached) {
if(!_EBCDICSwapLFNL(cnv->sharedData, pErrorCode)) {
if(U_FAILURE(*pErrorCode)) {
return; /* something went wrong */
}
/* the option does not apply, remove it */
cnv->options=pArgs->options&=~UCNV_OPTION_SWAP_LFNL;
}
}
}
if(uprv_strstr(pArgs->name, "18030")!=NULL) {
if(uprv_strstr(pArgs->name, "gb18030")!=NULL || uprv_strstr(pArgs->name, "GB18030")!=NULL) {
/* set a flag for GB 18030 mode, which changes the callback behavior */
cnv->options|=_MBCS_OPTION_GB18030;
}
} else if((uprv_strstr(pArgs->name, "KEIS")!=NULL) || (uprv_strstr(pArgs->name, "keis")!=NULL)) {
/* set a flag for KEIS converter, which changes the SI/SO character sequence */
cnv->options|=_MBCS_OPTION_KEIS;
} else if((uprv_strstr(pArgs->name, "JEF")!=NULL) || (uprv_strstr(pArgs->name, "jef")!=NULL)) {
/* set a flag for JEF converter, which changes the SI/SO character sequence */
cnv->options|=_MBCS_OPTION_JEF;
} else if((uprv_strstr(pArgs->name, "JIPS")!=NULL) || (uprv_strstr(pArgs->name, "jips")!=NULL)) {
/* set a flag for JIPS converter, which changes the SI/SO character sequence */
cnv->options|=_MBCS_OPTION_JIPS;
}
/* fix maxBytesPerUChar depending on outputType and options etc. */
if(outputType==MBCS_OUTPUT_2_SISO) {
cnv->maxBytesPerUChar=3; /* SO+DBCS */
}
extIndexes=mbcsTable->extIndexes;
if(extIndexes!=NULL) {
maxBytesPerUChar=(int8_t)UCNV_GET_MAX_BYTES_PER_UCHAR(extIndexes);
if(outputType==MBCS_OUTPUT_2_SISO) {
++maxBytesPerUChar; /* SO + multiple DBCS */
}
if(maxBytesPerUChar>cnv->maxBytesPerUChar) {
cnv->maxBytesPerUChar=maxBytesPerUChar;
}
}
#if 0
/*
* documentation of UConverter fields used for status
* all of these fields are (re)set to 0 by ucnv_bld.c and ucnv_reset()
*/
/* toUnicode */
cnv->toUnicodeStatus=0; /* offset */
cnv->mode=0; /* state */
cnv->toULength=0; /* byteIndex */
/* fromUnicode */
cnv->fromUChar32=0;
cnv->fromUnicodeStatus=1; /* prevLength */
#endif
}
static const char *
ucnv_MBCSGetName(const UConverter *cnv) {
if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0 && cnv->sharedData->mbcs.swapLFNLName!=NULL) {
return cnv->sharedData->mbcs.swapLFNLName;
} else {
return cnv->sharedData->staticData->name;
}
}
/* MBCS-to-Unicode conversion functions ------------------------------------- */
static UChar32
ucnv_MBCSGetFallback(UConverterMBCSTable *mbcsTable, uint32_t offset) {
const _MBCSToUFallback *toUFallbacks;
uint32_t i, start, limit;
limit=mbcsTable->countToUFallbacks;
if(limit>0) {
/* do a binary search for the fallback mapping */
toUFallbacks=mbcsTable->toUFallbacks;
start=0;
while(start<limit-1) {
i=(start+limit)/2;
if(offset<toUFallbacks[i].offset) {
limit=i;
} else {
start=i;
}
}
/* did we really find it? */
if(offset==toUFallbacks[start].offset) {
return toUFallbacks[start].codePoint;
}
}
return 0xfffe;
}
/* This version of ucnv_MBCSToUnicodeWithOffsets() is optimized for single-byte, single-state codepages. */
static void
ucnv_MBCSSingleToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs,
UErrorCode *pErrorCode) {
UConverter *cnv;
const uint8_t *source, *sourceLimit;
UChar *target;
const UChar *targetLimit;
int32_t *offsets;
const int32_t (*stateTable)[256];
int32_t sourceIndex;
int32_t entry;
UChar c;
uint8_t action;
/* set up the local pointers */
cnv=pArgs->converter;
source=(const uint8_t *)pArgs->source;
sourceLimit=(const uint8_t *)pArgs->sourceLimit;
target=pArgs->target;
targetLimit=pArgs->targetLimit;
offsets=pArgs->offsets;
if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable;
} else {
stateTable=cnv->sharedData->mbcs.stateTable;
}
/* sourceIndex=-1 if the current character began in the previous buffer */
sourceIndex=0;
/* conversion loop */
while(source<sourceLimit) {
/*
* This following test is to see if available input would overflow the output.
* It does not catch output of more than one code unit that
* overflows as a result of a surrogate pair or callback output
* from the last source byte.
* Therefore, those situations also test for overflows and will
* then break the loop, too.
*/
if(target>=targetLimit) {
/* target is full */
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
break;
}
entry=stateTable[0][*source++];
/* MBCS_ENTRY_IS_FINAL(entry) */
/* test the most common case first */
if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) {
/* output BMP code point */
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
/* normal end of action codes: prepare for a new character */
++sourceIndex;
continue;
}
/*
* An if-else-if chain provides more reliable performance for
* the most common cases compared to a switch.
*/
action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
if(action==MBCS_STATE_VALID_DIRECT_20 ||
(action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv))
) {
entry=MBCS_ENTRY_FINAL_VALUE(entry);
/* output surrogate pair */
*target++=(UChar)(0xd800|(UChar)(entry>>10));
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
c=(UChar)(0xdc00|(UChar)(entry&0x3ff));
if(target<targetLimit) {
*target++=c;
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
} else {
/* target overflow */
cnv->UCharErrorBuffer[0]=c;
cnv->UCharErrorBufferLength=1;
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
break;
}
++sourceIndex;
continue;
} else if(action==MBCS_STATE_FALLBACK_DIRECT_16) {
if(UCNV_TO_U_USE_FALLBACK(cnv)) {
/* output BMP code point */
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
++sourceIndex;
continue;
}
} else if(action==MBCS_STATE_UNASSIGNED) {
/* just fall through */
} else if(action==MBCS_STATE_ILLEGAL) {
/* callback(illegal) */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
} else {
/* reserved, must never occur */
++sourceIndex;
continue;
}
if(U_FAILURE(*pErrorCode)) {
/* callback(illegal) */
break;
} else /* unassigned sequences indicated with byteIndex>0 */ {
/* try an extension mapping */
pArgs->source=(const char *)source;
cnv->toUBytes[0]=*(source-1);
cnv->toULength=_extToU(cnv, cnv->sharedData,
1, &source, sourceLimit,
&target, targetLimit,
&offsets, sourceIndex,
pArgs->flush,
pErrorCode);
sourceIndex+=1+(int32_t)(source-(const uint8_t *)pArgs->source);
if(U_FAILURE(*pErrorCode)) {
/* not mappable or buffer overflow */
break;
}
}
}
/* write back the updated pointers */
pArgs->source=(const char *)source;
pArgs->target=target;
pArgs->offsets=offsets;
}
/*
* This version of ucnv_MBCSSingleToUnicodeWithOffsets() is optimized for single-byte, single-state codepages
* that only map to and from the BMP.
* In addition to single-byte optimizations, the offset calculations
* become much easier.
*/
static void
ucnv_MBCSSingleToBMPWithOffsets(UConverterToUnicodeArgs *pArgs,
UErrorCode *pErrorCode) {
UConverter *cnv;
const uint8_t *source, *sourceLimit, *lastSource;
UChar *target;
int32_t targetCapacity, length;
int32_t *offsets;
const int32_t (*stateTable)[256];
int32_t sourceIndex;
int32_t entry;
uint8_t action;
/* set up the local pointers */
cnv=pArgs->converter;
source=(const uint8_t *)pArgs->source;
sourceLimit=(const uint8_t *)pArgs->sourceLimit;
target=pArgs->target;
targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target);
offsets=pArgs->offsets;
if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable;
} else {
stateTable=cnv->sharedData->mbcs.stateTable;
}
/* sourceIndex=-1 if the current character began in the previous buffer */
sourceIndex=0;
lastSource=source;
/*
* since the conversion here is 1:1 UChar:uint8_t, we need only one counter
* for the minimum of the sourceLength and targetCapacity
*/
length=(int32_t)(sourceLimit-source);
if(length<targetCapacity) {
targetCapacity=length;
}
#if MBCS_UNROLL_SINGLE_TO_BMP
/* unrolling makes it faster on Pentium III/Windows 2000 */
/* unroll the loop with the most common case */
unrolled:
if(targetCapacity>=16) {
int32_t count, loops, oredEntries;
loops=count=targetCapacity>>4;
do {
oredEntries=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
oredEntries|=entry=stateTable[0][*source++];
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
/* were all 16 entries really valid? */
if(!MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(oredEntries)) {
/* no, return to the first of these 16 */
source-=16;
target-=16;
break;
}
} while(--count>0);
count=loops-count;
targetCapacity-=16*count;
if(offsets!=NULL) {
lastSource+=16*count;
while(count>0) {
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
*offsets++=sourceIndex++;
--count;
}
}
}
#endif
/* conversion loop */
while(targetCapacity > 0 && source < sourceLimit) {
entry=stateTable[0][*source++];
/* MBCS_ENTRY_IS_FINAL(entry) */
/* test the most common case first */
if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) {
/* output BMP code point */
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
--targetCapacity;
continue;
}
/*
* An if-else-if chain provides more reliable performance for
* the most common cases compared to a switch.
*/
action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
if(action==MBCS_STATE_FALLBACK_DIRECT_16) {
if(UCNV_TO_U_USE_FALLBACK(cnv)) {
/* output BMP code point */
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
--targetCapacity;
continue;
}
} else if(action==MBCS_STATE_UNASSIGNED) {
/* just fall through */
} else if(action==MBCS_STATE_ILLEGAL) {
/* callback(illegal) */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
} else {
/* reserved, must never occur */
continue;
}
/* set offsets since the start or the last extension */
if(offsets!=NULL) {
int32_t count=(int32_t)(source-lastSource);
/* predecrement: do not set the offset for the callback-causing character */
while(--count>0) {
*offsets++=sourceIndex++;
}
/* offset and sourceIndex are now set for the current character */
}
if(U_FAILURE(*pErrorCode)) {
/* callback(illegal) */
break;
} else /* unassigned sequences indicated with byteIndex>0 */ {
/* try an extension mapping */
lastSource=source;
cnv->toUBytes[0]=*(source-1);
cnv->toULength=_extToU(cnv, cnv->sharedData,
1, &source, sourceLimit,
&target, pArgs->targetLimit,
&offsets, sourceIndex,
pArgs->flush,
pErrorCode);
sourceIndex+=1+(int32_t)(source-lastSource);
if(U_FAILURE(*pErrorCode)) {
/* not mappable or buffer overflow */
break;
}
/* recalculate the targetCapacity after an extension mapping */
targetCapacity=(int32_t)(pArgs->targetLimit-target);
length=(int32_t)(sourceLimit-source);
if(length<targetCapacity) {
targetCapacity=length;
}
}
#if MBCS_UNROLL_SINGLE_TO_BMP
/* unrolling makes it faster on Pentium III/Windows 2000 */
goto unrolled;
#endif
}
if(U_SUCCESS(*pErrorCode) && source<sourceLimit && target>=pArgs->targetLimit) {
/* target is full */
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
}
/* set offsets since the start or the last callback */
if(offsets!=NULL) {
size_t count=source-lastSource;
while(count>0) {
*offsets++=sourceIndex++;
--count;
}
}
/* write back the updated pointers */
pArgs->source=(const char *)source;
pArgs->target=target;
pArgs->offsets=offsets;
}
static UBool
hasValidTrailBytes(const int32_t (*stateTable)[256], uint8_t state) {
const int32_t *row=stateTable[state];
int32_t b, entry;
/* First test for final entries in this state for some commonly valid byte values. */
entry=row[0xa1];
if( !MBCS_ENTRY_IS_TRANSITION(entry) &&
MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL
) {
return TRUE;
}
entry=row[0x41];
if( !MBCS_ENTRY_IS_TRANSITION(entry) &&
MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL
) {
return TRUE;
}
/* Then test for final entries in this state. */
for(b=0; b<=0xff; ++b) {
entry=row[b];
if( !MBCS_ENTRY_IS_TRANSITION(entry) &&
MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL
) {
return TRUE;
}
}
/* Then recurse for transition entries. */
for(b=0; b<=0xff; ++b) {
entry=row[b];
if( MBCS_ENTRY_IS_TRANSITION(entry) &&
hasValidTrailBytes(stateTable, (uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry))
) {
return TRUE;
}
}
return FALSE;
}
/*
* Is byte b a single/lead byte in this state?
* Recurse for transition states, because here we don't want to say that
* b is a lead byte if all byte sequences that start with b are illegal.
*/
static UBool
isSingleOrLead(const int32_t (*stateTable)[256], uint8_t state, UBool isDBCSOnly, uint8_t b) {
const int32_t *row=stateTable[state];
int32_t entry=row[b];
if(MBCS_ENTRY_IS_TRANSITION(entry)) { /* lead byte */
return hasValidTrailBytes(stateTable, (uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry));
} else {
uint8_t action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
if(action==MBCS_STATE_CHANGE_ONLY && isDBCSOnly) {
return FALSE; /* SI/SO are illegal for DBCS-only conversion */
} else {
return action!=MBCS_STATE_ILLEGAL;
}
}
}
U_CFUNC void
ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs,
UErrorCode *pErrorCode) {
UConverter *cnv;
const uint8_t *source, *sourceLimit;
UChar *target;
const UChar *targetLimit;
int32_t *offsets;
const int32_t (*stateTable)[256];
const uint16_t *unicodeCodeUnits;
uint32_t offset;
uint8_t state;
int8_t byteIndex;
uint8_t *bytes;
int32_t sourceIndex, nextSourceIndex;
int32_t entry;
UChar c;
uint8_t action;
/* use optimized function if possible */
cnv=pArgs->converter;
if(cnv->preToULength>0) {
/*
* pass sourceIndex=-1 because we continue from an earlier buffer
* in the future, this may change with continuous offsets
*/
ucnv_extContinueMatchToU(cnv, pArgs, -1, pErrorCode);
if(U_FAILURE(*pErrorCode) || cnv->preToULength<0) {
return;
}
}
if(cnv->sharedData->mbcs.countStates==1) {
if(!(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) {
ucnv_MBCSSingleToBMPWithOffsets(pArgs, pErrorCode);
} else {
ucnv_MBCSSingleToUnicodeWithOffsets(pArgs, pErrorCode);
}
return;
}
/* set up the local pointers */
source=(const uint8_t *)pArgs->source;
sourceLimit=(const uint8_t *)pArgs->sourceLimit;
target=pArgs->target;
targetLimit=pArgs->targetLimit;
offsets=pArgs->offsets;
if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable;
} else {
stateTable=cnv->sharedData->mbcs.stateTable;
}
unicodeCodeUnits=cnv->sharedData->mbcs.unicodeCodeUnits;
/* get the converter state from UConverter */
offset=cnv->toUnicodeStatus;
byteIndex=cnv->toULength;
bytes=cnv->toUBytes;
/*
* if we are in the SBCS state for a DBCS-only converter,
* then load the DBCS state from the MBCS data
* (dbcsOnlyState==0 if it is not a DBCS-only converter)
*/
if((state=(uint8_t)(cnv->mode))==0) {
state=cnv->sharedData->mbcs.dbcsOnlyState;
}
/* sourceIndex=-1 if the current character began in the previous buffer */
sourceIndex=byteIndex==0 ? 0 : -1;
nextSourceIndex=0;
/* conversion loop */
while(source<sourceLimit) {
/*
* This following test is to see if available input would overflow the output.
* It does not catch output of more than one code unit that
* overflows as a result of a surrogate pair or callback output
* from the last source byte.
* Therefore, those situations also test for overflows and will
* then break the loop, too.
*/
if(target>=targetLimit) {
/* target is full */
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
break;
}
if(byteIndex==0) {
/* optimized loop for 1/2-byte input and BMP output */
if(offsets==NULL) {
do {
entry=stateTable[state][*source];
if(MBCS_ENTRY_IS_TRANSITION(entry)) {
state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry);
offset=MBCS_ENTRY_TRANSITION_OFFSET(entry);
++source;
if( source<sourceLimit &&
MBCS_ENTRY_IS_FINAL(entry=stateTable[state][*source]) &&
MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16 &&
(c=unicodeCodeUnits[offset+MBCS_ENTRY_FINAL_VALUE_16(entry)])<0xfffe
) {
++source;
*target++=c;
state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
offset=0;
} else {
/* set the state and leave the optimized loop */
bytes[0]=*(source-1);
byteIndex=1;
break;
}
} else {
if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) {
/* output BMP code point */
++source;
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
} else {
/* leave the optimized loop */
break;
}
}
} while(source<sourceLimit && target<targetLimit);
} else /* offsets!=NULL */ {
do {
entry=stateTable[state][*source];
if(MBCS_ENTRY_IS_TRANSITION(entry)) {
state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry);
offset=MBCS_ENTRY_TRANSITION_OFFSET(entry);
++source;
if( source<sourceLimit &&
MBCS_ENTRY_IS_FINAL(entry=stateTable[state][*source]) &&
MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16 &&
(c=unicodeCodeUnits[offset+MBCS_ENTRY_FINAL_VALUE_16(entry)])<0xfffe
) {
++source;
*target++=c;
if(offsets!=NULL) {
*offsets++=sourceIndex;
sourceIndex=(nextSourceIndex+=2);
}
state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
offset=0;
} else {
/* set the state and leave the optimized loop */
++nextSourceIndex;
bytes[0]=*(source-1);
byteIndex=1;
break;
}
} else {
if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) {
/* output BMP code point */
++source;
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
if(offsets!=NULL) {
*offsets++=sourceIndex;
sourceIndex=++nextSourceIndex;
}
state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
} else {
/* leave the optimized loop */
break;
}
}
} while(source<sourceLimit && target<targetLimit);
}
/*
* these tests and break statements could be put inside the loop
* if C had "break outerLoop" like Java
*/
if(source>=sourceLimit) {
break;
}
if(target>=targetLimit) {
/* target is full */
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
break;
}
++nextSourceIndex;
bytes[byteIndex++]=*source++;
} else /* byteIndex>0 */ {
++nextSourceIndex;
entry=stateTable[state][bytes[byteIndex++]=*source++];
}
if(MBCS_ENTRY_IS_TRANSITION(entry)) {
state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry);
offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry);
continue;
}
/* save the previous state for proper extension mapping with SI/SO-stateful converters */
cnv->mode=state;
/* set the next state early so that we can reuse the entry variable */
state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
/*
* An if-else-if chain provides more reliable performance for
* the most common cases compared to a switch.
*/
action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
if(action==MBCS_STATE_VALID_16) {
offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
c=unicodeCodeUnits[offset];
if(c<0xfffe) {
/* output BMP code point */
*target++=c;
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
byteIndex=0;
} else if(c==0xfffe) {
if(UCNV_TO_U_USE_FALLBACK(cnv) && (entry=(int32_t)ucnv_MBCSGetFallback(&cnv->sharedData->mbcs, offset))!=0xfffe) {
/* output fallback BMP code point */
*target++=(UChar)entry;
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
byteIndex=0;
}
} else {
/* callback(illegal) */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
}
} else if(action==MBCS_STATE_VALID_DIRECT_16) {
/* output BMP code point */
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
byteIndex=0;
} else if(action==MBCS_STATE_VALID_16_PAIR) {
offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
c=unicodeCodeUnits[offset++];
if(c<0xd800) {
/* output BMP code point below 0xd800 */
*target++=c;
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
byteIndex=0;
} else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) {
/* output roundtrip or fallback surrogate pair */
*target++=(UChar)(c&0xdbff);
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
byteIndex=0;
if(target<targetLimit) {
*target++=unicodeCodeUnits[offset];
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
} else {
/* target overflow */
cnv->UCharErrorBuffer[0]=unicodeCodeUnits[offset];
cnv->UCharErrorBufferLength=1;
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
offset=0;
break;
}
} else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) {
/* output roundtrip BMP code point above 0xd800 or fallback BMP code point */
*target++=unicodeCodeUnits[offset];
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
byteIndex=0;
} else if(c==0xffff) {
/* callback(illegal) */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
}
} else if(action==MBCS_STATE_VALID_DIRECT_20 ||
(action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv))
) {
entry=MBCS_ENTRY_FINAL_VALUE(entry);
/* output surrogate pair */
*target++=(UChar)(0xd800|(UChar)(entry>>10));
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
byteIndex=0;
c=(UChar)(0xdc00|(UChar)(entry&0x3ff));
if(target<targetLimit) {
*target++=c;
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
} else {
/* target overflow */
cnv->UCharErrorBuffer[0]=c;
cnv->UCharErrorBufferLength=1;
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
offset=0;
break;
}
} else if(action==MBCS_STATE_CHANGE_ONLY) {
/*
* This serves as a state change without any output.
* It is useful for reading simple stateful encodings,
* for example using just Shift-In/Shift-Out codes.
* The 21 unused bits may later be used for more sophisticated
* state transitions.
*/
if(cnv->sharedData->mbcs.dbcsOnlyState==0) {
byteIndex=0;
} else {
/* SI/SO are illegal for DBCS-only conversion */
state=(uint8_t)(cnv->mode); /* restore the previous state */
/* callback(illegal) */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
}
} else if(action==MBCS_STATE_FALLBACK_DIRECT_16) {
if(UCNV_TO_U_USE_FALLBACK(cnv)) {
/* output BMP code point */
*target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
if(offsets!=NULL) {
*offsets++=sourceIndex;
}
byteIndex=0;
}
} else if(action==MBCS_STATE_UNASSIGNED) {
/* just fall through */
} else if(action==MBCS_STATE_ILLEGAL) {
/* callback(illegal) */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
} else {
/* reserved, must never occur */
byteIndex=0;
}
/* end of action codes: prepare for a new character */
offset=0;
if(byteIndex==0) {
sourceIndex=nextSourceIndex;
} else if(U_FAILURE(*pErrorCode)) {
/* callback(illegal) */
if(byteIndex>1) {
/*
* Ticket 5691: consistent illegal sequences:
* - We include at least the first byte in the illegal sequence.
* - If any of the non-initial bytes could be the start of a character,
* we stop the illegal sequence before the first one of those.
*/
UBool isDBCSOnly=(UBool)(cnv->sharedData->mbcs.dbcsOnlyState!=0);
int8_t i;
for(i=1;
i<byteIndex && !isSingleOrLead(stateTable, state, isDBCSOnly, bytes[i]);
++i) {}
if(i<byteIndex) {
/* Back out some bytes. */
int8_t backOutDistance=byteIndex-i;
int32_t bytesFromThisBuffer=(int32_t)(source-(const uint8_t *)pArgs->source);
byteIndex=i; /* length of reported illegal byte sequence */
if(backOutDistance<=bytesFromThisBuffer) {
source-=backOutDistance;
} else {
/* Back out bytes from the previous buffer: Need to replay them. */
cnv->preToULength=(int8_t)(bytesFromThisBuffer-backOutDistance);
/* preToULength is negative! */
uprv_memcpy(cnv->preToU, bytes+i, -cnv->preToULength);
source=(const uint8_t *)pArgs->source;
}
}
}
break;
} else /* unassigned sequences indicated with byteIndex>0 */ {
/* try an extension mapping */
pArgs->source=(const char *)source;
byteIndex=_extToU(cnv, cnv->sharedData,
byteIndex, &source, sourceLimit,
&target, targetLimit,
&offsets, sourceIndex,
pArgs->flush,
pErrorCode);
sourceIndex=nextSourceIndex+=(int32_t)(source-(const uint8_t *)pArgs->source);
if(U_FAILURE(*pErrorCode)) {
/* not mappable or buffer overflow */
break;
}
}
}
/* set the converter state back into UConverter */
cnv->toUnicodeStatus=offset;
cnv->mode=state;
cnv->toULength=byteIndex;
/* write back the updated pointers */
pArgs->source=(const char *)source;
pArgs->target=target;
pArgs->offsets=offsets;
}
/*
* This version of ucnv_MBCSGetNextUChar() is optimized for single-byte, single-state codepages.
* We still need a conversion loop in case we find reserved action codes, which are to be ignored.
*/
static UChar32
ucnv_MBCSSingleGetNextUChar(UConverterToUnicodeArgs *pArgs,
UErrorCode *pErrorCode) {
UConverter *cnv;
const int32_t (*stateTable)[256];
const uint8_t *source, *sourceLimit;
int32_t entry;
uint8_t action;
/* set up the local pointers */
cnv=pArgs->converter;
source=(const uint8_t *)pArgs->source;
sourceLimit=(const uint8_t *)pArgs->sourceLimit;
if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable;
} else {
stateTable=cnv->sharedData->mbcs.stateTable;
}
/* conversion loop */
while(source<sourceLimit) {
entry=stateTable[0][*source++];
/* MBCS_ENTRY_IS_FINAL(entry) */
/* write back the updated pointer early so that we can return directly */
pArgs->source=(const char *)source;
if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) {
/* output BMP code point */
return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
}
/*
* An if-else-if chain provides more reliable performance for
* the most common cases compared to a switch.
*/
action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
if( action==MBCS_STATE_VALID_DIRECT_20 ||
(action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv))
) {
/* output supplementary code point */
return (UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000);
} else if(action==MBCS_STATE_FALLBACK_DIRECT_16) {
if(UCNV_TO_U_USE_FALLBACK(cnv)) {
/* output BMP code point */
return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
}
} else if(action==MBCS_STATE_UNASSIGNED) {
/* just fall through */
} else if(action==MBCS_STATE_ILLEGAL) {
/* callback(illegal) */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
} else {
/* reserved, must never occur */
continue;
}
if(U_FAILURE(*pErrorCode)) {
/* callback(illegal) */
break;
} else /* unassigned sequence */ {
/* defer to the generic implementation */
pArgs->source=(const char *)source-1;
return UCNV_GET_NEXT_UCHAR_USE_TO_U;
}
}
/* no output because of empty input or only state changes */
*pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
return 0xffff;
}
/*
* Version of _MBCSToUnicodeWithOffsets() optimized for single-character
* conversion without offset handling.
*
* When a character does not have a mapping to Unicode, then we return to the
* generic ucnv_getNextUChar() code for extension/GB 18030 and error/callback
* handling.
* We also defer to the generic code in other complicated cases and have them
* ultimately handled by _MBCSToUnicodeWithOffsets() itself.
*
* All normal mappings and errors are handled here.
*/
static UChar32
ucnv_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs,
UErrorCode *pErrorCode) {
UConverter *cnv;
const uint8_t *source, *sourceLimit, *lastSource;
const int32_t (*stateTable)[256];
const uint16_t *unicodeCodeUnits;
uint32_t offset;
uint8_t state;
int32_t entry;
UChar32 c;
uint8_t action;
/* use optimized function if possible */
cnv=pArgs->converter;
if(cnv->preToULength>0) {
/* use the generic code in ucnv_getNextUChar() to continue with a partial match */
return UCNV_GET_NEXT_UCHAR_USE_TO_U;
}
if(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SURROGATES) {
/*
* Using the generic ucnv_getNextUChar() code lets us deal correctly
* with the rare case of a codepage that maps single surrogates
* without adding the complexity to this already complicated function here.
*/
return UCNV_GET_NEXT_UCHAR_USE_TO_U;
} else if(cnv->sharedData->mbcs.countStates==1) {
return ucnv_MBCSSingleGetNextUChar(pArgs, pErrorCode);
}
/* set up the local pointers */
source=lastSource=(const uint8_t *)pArgs->source;
sourceLimit=(const uint8_t *)pArgs->sourceLimit;
if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) {
stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable;
} else {
stateTable=cnv->sharedData->mbcs.stateTable;
}
unicodeCodeUnits=cnv->sharedData->mbcs.unicodeCodeUnits;
/* get the converter state from UConverter */
offset=cnv->toUnicodeStatus;
/*
* if we are in the SBCS state for a DBCS-only converter,
* then load the DBCS state from the MBCS data
* (dbcsOnlyState==0 if it is not a DBCS-only converter)
*/
if((state=(uint8_t)(cnv->mode))==0) {
state=cnv->sharedData->mbcs.dbcsOnlyState;
}
/* conversion loop */
c=U_SENTINEL;
while(source<sourceLimit) {
entry=stateTable[state][*source++];
if(MBCS_ENTRY_IS_TRANSITION(entry)) {
state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry);
offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry);
/* optimization for 1/2-byte input and BMP output */
if( source<sourceLimit &&
MBCS_ENTRY_IS_FINAL(entry=stateTable[state][*source]) &&
MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16 &&
(c=unicodeCodeUnits[offset+MBCS_ENTRY_FINAL_VALUE_16(entry)])<0xfffe
) {
++source;
state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
/* output BMP code point */
break;
}
} else {
/* save the previous state for proper extension mapping with SI/SO-stateful converters */
cnv->mode=state;
/* set the next state early so that we can reuse the entry variable */
state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */
/*
* An if-else-if chain provides more reliable performance for
* the most common cases compared to a switch.
*/
action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry));
if(action==MBCS_STATE_VALID_DIRECT_16) {
/* output BMP code point */
c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
break;
} else if(action==MBCS_STATE_VALID_16) {
offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
c=unicodeCodeUnits[offset];
if(c<0xfffe) {
/* output BMP code point */
break;
} else if(c==0xfffe) {
if(UCNV_TO_U_USE_FALLBACK(cnv) && (c=ucnv_MBCSGetFallback(&cnv->sharedData->mbcs, offset))!=0xfffe) {
break;
}
} else {
/* callback(illegal) */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
}
} else if(action==MBCS_STATE_VALID_16_PAIR) {
offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
c=unicodeCodeUnits[offset++];
if(c<0xd800) {
/* output BMP code point below 0xd800 */
break;
} else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) {
/* output roundtrip or fallback supplementary code point */
c=((c&0x3ff)<<10)+unicodeCodeUnits[offset]+(0x10000-0xdc00);
break;
} else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) {
/* output roundtrip BMP code point above 0xd800 or fallback BMP code point */
c=unicodeCodeUnits[offset];
break;
} else if(c==0xffff) {
/* callback(illegal) */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
}
} else if(action==MBCS_STATE_VALID_DIRECT_20 ||
(action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv))
) {
/* output supplementary code point */
c=(UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000);
break;
} else if(action==MBCS_STATE_CHANGE_ONLY) {
/*
* This serves as a state change without any output.
* It is useful for reading simple stateful encodings,
* for example using just Shift-In/Shift-Out codes.
* The 21 unused bits may later be used for more sophisticated
* state transitions.
*/
if(cnv->sharedData->mbcs.dbcsOnlyState!=0) {
/* SI/SO are illegal for DBCS-only conversion */
state=(uint8_t)(cnv->mode); /* restore the previous state */
/* callback(illegal) */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
}
} else if(action==MBCS_STATE_FALLBACK_DIRECT_16) {
if(UCNV_TO_U_USE_FALLBACK(cnv)) {
/* output BMP code point */
c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry);
break;
}
} else if(action==MBCS_STATE_UNASSIGNED) {
/* just fall through */
} else if(action==MBCS_STATE_ILLEGAL) {
/* callback(illegal) */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
} else {
/* reserved (must never occur), or only state change */
offset=0;
lastSource=source;
continue;
}
/* end of action codes: prepare for a new character */
offset=0;
if(U_FAILURE(*pErrorCode)) {
/* callback(illegal) */
break;
} else /* unassigned sequence */ {
/* defer to the generic implementation */
cnv->toUnicodeStatus=0;
cnv->mode=state;
pArgs->source=(const char *)lastSource;
return UCNV_GET_NEXT_UCHAR_USE_TO_U;
}
}
}
if(c<0) {
if(U_SUCCESS(*pErrorCode) && source==sourceLimit && lastSource<source) {
/* incomplete character byte sequence */
uint8_t *bytes=cnv->toUBytes;
cnv->toULength=(int8_t)(source-lastSource);
do {
*bytes++=*lastSource++;
} while(lastSource<source);
*pErrorCode=U_TRUNCATED_CHAR_FOUND;
} else if(U_FAILURE(*pErrorCode)) {
/* callback(illegal) */
/*
* Ticket 5691: consistent illegal sequences:
* - We include at least the first byte in the illegal sequence.
* - If any of the non-initial bytes could be the start of a character,
* we stop the illegal sequence before the first one of those.
*/
UBool isDBCSOnly=(UBool)(cnv->sharedData->mbcs.dbcsOnlyState!=0);
uint8_t *bytes=cnv->toUBytes;
*bytes++=*lastSource++; /* first byte */
if(lastSource==source) {
cnv->toULength=1;
} else /* lastSource<source: multi-byte character */ {
int8_t i;
for(i=1;
lastSource<source && !isSingleOrLead(stateTable, state, isDBCSOnly, *lastSource);
++i
) {
*bytes++=*lastSource++;
}
cnv->toULength=i;
source=lastSource;
}
} else {
/* no output because of empty input or only state changes */
*pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
}