| /* |
| ******************************************************************************* |
| * |
| * Copyright (C) 2003-2006, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| * |
| ******************************************************************************* |
| * file name: uit_len8.c |
| * encoding: US-ASCII |
| * tab size: 8 (not used) |
| * indentation:4 |
| * |
| * created on: 2003feb10 |
| * created by: Markus W. Scherer |
| * |
| * This file contains the implementation of the "lenient UTF-8" UCharIterator |
| * as used in the uciter8 sample code. |
| * UTF-8-style macros are defined as well as the UCharIterator. |
| * The macros are incomplete (do not assemble code points from pairs of |
| * surrogates, see comment below) |
| * but sufficient for the iterator. |
| */ |
| |
| #include <string.h> |
| #include "unicode/utypes.h" |
| #include "unicode/uiter.h" |
| |
| /* lenient UTF-8/CESU-8 macros ---------------------------------------------- */ |
| |
| /* |
| * This code leniently reads 8-bit Unicode strings, |
| * which could contain a mix of UTF-8 and CESU-8. |
| * More precisely: |
| * - supplementary code points may be encoded with dedicated 4-byte sequences |
| * (UTF-8 style) |
| * - supplementary code points may be encoded with |
| * pairs of 3-byte sequences, one for each surrogate of the UTF-16 form |
| * (CESU-8 style) |
| * - single surrogates are allowed, encoded with their "natural" 3-byte sequences |
| * |
| * Limitation: |
| * Right now, the macros do not attempt to assemble code points from pairs of |
| * separately encoded surrogates. |
| * This would not be sufficient for processing based on these macros, |
| * but it is sufficient for a UCharIterator that returns only UChars anyway. |
| * |
| * The code is copied and modified from utf_impl.c and utf8.h. |
| * |
| * Change 2006feb08: Much of the implementation code is replaced by calling |
| * the utf_impl.c functions which accept a new "strict" parameter value |
| * of -2 implementing exactly this leniency. |
| */ |
| |
| #define L8_NEXT(s, i, length, c) { \ |
| (c)=(uint8_t)(s)[(i)++]; \ |
| if((c)>=0x80) { \ |
| if(U8_IS_LEAD(c)) { \ |
| (c)=utf8_nextCharSafeBody((const uint8_t *)s, &(i), (int32_t)(length), c, -2); \ |
| } else { \ |
| (c)=U_SENTINEL; \ |
| } \ |
| } \ |
| } |
| |
| #define L8_PREV(s, start, i, c) { \ |
| (c)=(uint8_t)(s)[--(i)]; \ |
| if((c)>=0x80) { \ |
| if((c)<=0xbf) { \ |
| (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -2); \ |
| } else { \ |
| (c)=U_SENTINEL; \ |
| } \ |
| } \ |
| } |
| |
| /* lenient-8 UCharIterator -------------------------------------------------- */ |
| |
| /* |
| * This is a copy of the UTF-8 UCharIterator in uiter.cpp, |
| * except that it uses the lenient-8-bit-Unicode macros above. |
| */ |
| |
| /* |
| * Minimal implementation: |
| * Maintain a single-UChar buffer for an additional surrogate. |
| * The caller must not modify start and limit because they are used internally. |
| * |
| * Use UCharIterator fields as follows: |
| * context pointer to UTF-8 string |
| * length UTF-16 length of the string; -1 until lazy evaluation |
| * start current UTF-8 index |
| * index current UTF-16 index; may be -1="unknown" after setState() |
| * limit UTF-8 length of the string |
| * reservedField supplementary code point |
| * |
| * Since UCharIterator delivers 16-bit code units, the iteration can be |
| * currently in the middle of the byte sequence for a supplementary code point. |
| * In this case, reservedField will contain that code point and start will |
| * point to after the corresponding byte sequence. The UTF-16 index will be |
| * one less than what it would otherwise be corresponding to the UTF-8 index. |
| * Otherwise, reservedField will be 0. |
| */ |
| |
| /* |
| * Possible optimization for NUL-terminated UTF-8 and UTF-16 strings: |
| * Add implementations that do not call strlen() for iteration but check for NUL. |
| */ |
| |
| static int32_t U_CALLCONV |
| lenient8IteratorGetIndex(UCharIterator *iter, UCharIteratorOrigin origin) { |
| switch(origin) { |
| case UITER_ZERO: |
| case UITER_START: |
| return 0; |
| case UITER_CURRENT: |
| if(iter->index<0) { |
| /* the current UTF-16 index is unknown after setState(), count from the beginning */ |
| const uint8_t *s; |
| UChar32 c; |
| int32_t i, limit, index; |
| |
| s=(const uint8_t *)iter->context; |
| i=index=0; |
| limit=iter->start; /* count up to the UTF-8 index */ |
| while(i<limit) { |
| L8_NEXT(s, i, limit, c); |
| if(c<=0xffff) { |
| ++index; |
| } else { |
| index+=2; |
| } |
| } |
| |
| iter->start=i; /* just in case setState() did not get us to a code point boundary */ |
| if(i==iter->limit) { |
| iter->length=index; /* in case it was <0 or wrong */ |
| } |
| if(iter->reservedField!=0) { |
| --index; /* we are in the middle of a supplementary code point */ |
| } |
| iter->index=index; |
| } |
| return iter->index; |
| case UITER_LIMIT: |
| case UITER_LENGTH: |
| if(iter->length<0) { |
| const uint8_t *s; |
| UChar32 c; |
| int32_t i, limit, length; |
| |
| s=(const uint8_t *)iter->context; |
| if(iter->index<0) { |
| /* |
| * the current UTF-16 index is unknown after setState(), |
| * we must first count from the beginning to here |
| */ |
| i=length=0; |
| limit=iter->start; |
| |
| /* count from the beginning to the current index */ |
| while(i<limit) { |
| L8_NEXT(s, i, limit, c); |
| if(c<=0xffff) { |
| ++length; |
| } else { |
| length+=2; |
| } |
| } |
| |
| /* assume i==limit==iter->start, set the UTF-16 index */ |
| iter->start=i; /* just in case setState() did not get us to a code point boundary */ |
| iter->index= iter->reservedField!=0 ? length-1 : length; |
| } else { |
| i=iter->start; |
| length=iter->index; |
| if(iter->reservedField!=0) { |
| ++length; |
| } |
| } |
| |
| /* count from the current index to the end */ |
| limit=iter->limit; |
| while(i<limit) { |
| L8_NEXT(s, i, limit, c); |
| if(c<=0xffff) { |
| ++length; |
| } else { |
| length+=2; |
| } |
| } |
| iter->length=length; |
| } |
| return iter->length; |
| default: |
| /* not a valid origin */ |
| /* Should never get here! */ |
| return -1; |
| } |
| } |
| |
| static int32_t U_CALLCONV |
| lenient8IteratorMove(UCharIterator *iter, int32_t delta, UCharIteratorOrigin origin) { |
| const uint8_t *s; |
| UChar32 c; |
| int32_t pos; /* requested UTF-16 index */ |
| int32_t i; /* UTF-8 index */ |
| UBool havePos; |
| |
| /* calculate the requested UTF-16 index */ |
| switch(origin) { |
| case UITER_ZERO: |
| case UITER_START: |
| pos=delta; |
| havePos=TRUE; |
| /* iter->index<0 (unknown) is possible */ |
| break; |
| case UITER_CURRENT: |
| if(iter->index>=0) { |
| pos=iter->index+delta; |
| havePos=TRUE; |
| } else { |
| /* the current UTF-16 index is unknown after setState(), use only delta */ |
| pos=0; |
| havePos=FALSE; |
| } |
| break; |
| case UITER_LIMIT: |
| case UITER_LENGTH: |
| if(iter->length>=0) { |
| pos=iter->length+delta; |
| havePos=TRUE; |
| } else { |
| /* pin to the end, avoid counting the length */ |
| iter->index=-1; |
| iter->start=iter->limit; |
| iter->reservedField=0; |
| if(delta>=0) { |
| return UITER_UNKNOWN_INDEX; |
| } else { |
| /* the current UTF-16 index is unknown, use only delta */ |
| pos=0; |
| havePos=FALSE; |
| } |
| } |
| break; |
| default: |
| return -1; /* Error */ |
| } |
| |
| if(havePos) { |
| /* shortcuts: pinning to the edges of the string */ |
| if(pos<=0) { |
| iter->index=iter->start=iter->reservedField=0; |
| return 0; |
| } else if(iter->length>=0 && pos>=iter->length) { |
| iter->index=iter->length; |
| iter->start=iter->limit; |
| iter->reservedField=0; |
| return iter->index; |
| } |
| |
| /* minimize the number of L8_NEXT/PREV operations */ |
| if(iter->index<0 || pos<iter->index/2) { |
| /* go forward from the start instead of backward from the current index */ |
| iter->index=iter->start=iter->reservedField=0; |
| } else if(iter->length>=0 && (iter->length-pos)<(pos-iter->index)) { |
| /* |
| * if we have the UTF-16 index and length and the new position is |
| * closer to the end than the current index, |
| * then go backward from the end instead of forward from the current index |
| */ |
| iter->index=iter->length; |
| iter->start=iter->limit; |
| iter->reservedField=0; |
| } |
| |
| delta=pos-iter->index; |
| if(delta==0) { |
| return iter->index; /* nothing to do */ |
| } |
| } else { |
| /* move relative to unknown UTF-16 index */ |
| if(delta==0) { |
| return UITER_UNKNOWN_INDEX; /* nothing to do */ |
| } else if(-delta>=iter->start) { |
| /* moving backwards by more UChars than there are UTF-8 bytes, pin to 0 */ |
| iter->index=iter->start=iter->reservedField=0; |
| return 0; |
| } else if(delta>=(iter->limit-iter->start)) { |
| /* moving forward by more UChars than the remaining UTF-8 bytes, pin to the end */ |
| iter->index=iter->length; /* may or may not be <0 (unknown) */ |
| iter->start=iter->limit; |
| iter->reservedField=0; |
| return iter->index>=0 ? iter->index : UITER_UNKNOWN_INDEX; |
| } |
| } |
| |
| /* delta!=0 */ |
| |
| /* move towards the requested position, pin to the edges of the string */ |
| s=(const uint8_t *)iter->context; |
| pos=iter->index; /* could be <0 (unknown) */ |
| i=iter->start; |
| if(delta>0) { |
| /* go forward */ |
| int32_t limit=iter->limit; |
| if(iter->reservedField!=0) { |
| iter->reservedField=0; |
| ++pos; |
| --delta; |
| } |
| while(delta>0 && i<limit) { |
| L8_NEXT(s, i, limit, c); |
| if(c<0xffff) { |
| ++pos; |
| --delta; |
| } else if(delta>=2) { |
| pos+=2; |
| delta-=2; |
| } else /* delta==1 */ { |
| /* stop in the middle of a supplementary code point */ |
| iter->reservedField=c; |
| ++pos; |
| break; /* delta=0; */ |
| } |
| } |
| if(i==limit) { |
| if(iter->length<0 && iter->index>=0) { |
| iter->length= iter->reservedField==0 ? pos : pos+1; |
| } else if(iter->index<0 && iter->length>=0) { |
| iter->index= iter->reservedField==0 ? iter->length : iter->length-1; |
| } |
| } |
| } else /* delta<0 */ { |
| /* go backward */ |
| if(iter->reservedField!=0) { |
| iter->reservedField=0; |
| i-=4; /* we stayed behind the supplementary code point; go before it now */ |
| --pos; |
| ++delta; |
| } |
| while(delta<0 && i>0) { |
| L8_PREV(s, 0, i, c); |
| if(c<0xffff) { |
| --pos; |
| ++delta; |
| } else if(delta<=-2) { |
| pos-=2; |
| delta+=2; |
| } else /* delta==-1 */ { |
| /* stop in the middle of a supplementary code point */ |
| i+=4; /* back to behind this supplementary code point for consistent state */ |
| iter->reservedField=c; |
| --pos; |
| break; /* delta=0; */ |
| } |
| } |
| } |
| |
| iter->start=i; |
| if(iter->index>=0) { |
| return iter->index=pos; |
| } else { |
| /* we started with index<0 (unknown) so pos is bogus */ |
| if(i<=1) { |
| return iter->index=i; /* reached the beginning */ |
| } else { |
| /* we still don't know the UTF-16 index */ |
| return UITER_UNKNOWN_INDEX; |
| } |
| } |
| } |
| |
| static UBool U_CALLCONV |
| lenient8IteratorHasNext(UCharIterator *iter) { |
| return iter->reservedField!=0 || iter->start<iter->limit; |
| } |
| |
| static UBool U_CALLCONV |
| lenient8IteratorHasPrevious(UCharIterator *iter) { |
| return iter->start>0; |
| } |
| |
| static UChar32 U_CALLCONV |
| lenient8IteratorCurrent(UCharIterator *iter) { |
| if(iter->reservedField!=0) { |
| return U16_TRAIL(iter->reservedField); |
| } else if(iter->start<iter->limit) { |
| const uint8_t *s=(const uint8_t *)iter->context; |
| UChar32 c; |
| int32_t i=iter->start; |
| |
| L8_NEXT(s, i, iter->limit, c); |
| if(c<0) { |
| return 0xfffd; |
| } else if(c<=0xffff) { |
| return c; |
| } else { |
| return U16_LEAD(c); |
| } |
| } else { |
| return U_SENTINEL; |
| } |
| } |
| |
| static UChar32 U_CALLCONV |
| lenient8IteratorNext(UCharIterator *iter) { |
| int32_t index; |
| |
| if(iter->reservedField!=0) { |
| UChar trail=U16_TRAIL(iter->reservedField); |
| iter->reservedField=0; |
| if((index=iter->index)>=0) { |
| iter->index=index+1; |
| } |
| return trail; |
| } else if(iter->start<iter->limit) { |
| const uint8_t *s=(const uint8_t *)iter->context; |
| UChar32 c; |
| |
| L8_NEXT(s, iter->start, iter->limit, c); |
| if((index=iter->index)>=0) { |
| iter->index=++index; |
| if(iter->length<0 && iter->start==iter->limit) { |
| iter->length= c<=0xffff ? index : index+1; |
| } |
| } else if(iter->start==iter->limit && iter->length>=0) { |
| iter->index= c<=0xffff ? iter->length : iter->length-1; |
| } |
| if(c<0) { |
| return 0xfffd; |
| } else if(c<=0xffff) { |
| return c; |
| } else { |
| iter->reservedField=c; |
| return U16_LEAD(c); |
| } |
| } else { |
| return U_SENTINEL; |
| } |
| } |
| |
| static UChar32 U_CALLCONV |
| lenient8IteratorPrevious(UCharIterator *iter) { |
| int32_t index; |
| |
| if(iter->reservedField!=0) { |
| UChar lead=U16_LEAD(iter->reservedField); |
| iter->reservedField=0; |
| iter->start-=4; /* we stayed behind the supplementary code point; go before it now */ |
| if((index=iter->index)>0) { |
| iter->index=index-1; |
| } |
| return lead; |
| } else if(iter->start>0) { |
| const uint8_t *s=(const uint8_t *)iter->context; |
| UChar32 c; |
| |
| L8_PREV(s, 0, iter->start, c); |
| if((index=iter->index)>0) { |
| iter->index=index-1; |
| } else if(iter->start<=1) { |
| iter->index= c<=0xffff ? iter->start : iter->start+1; |
| } |
| if(c<0) { |
| return 0xfffd; |
| } else if(c<=0xffff) { |
| return c; |
| } else { |
| iter->start+=4; /* back to behind this supplementary code point for consistent state */ |
| iter->reservedField=c; |
| return U16_TRAIL(c); |
| } |
| } else { |
| return U_SENTINEL; |
| } |
| } |
| |
| static uint32_t U_CALLCONV |
| lenient8IteratorGetState(const UCharIterator *iter) { |
| uint32_t state=(uint32_t)(iter->start<<1); |
| if(iter->reservedField!=0) { |
| state|=1; |
| } |
| return state; |
| } |
| |
| static void U_CALLCONV |
| lenient8IteratorSetState(UCharIterator *iter, uint32_t state, UErrorCode *pErrorCode) { |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| /* do nothing */ |
| } else if(iter==NULL) { |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| } else if(state==lenient8IteratorGetState(iter)) { |
| /* setting to the current state: no-op */ |
| } else { |
| int32_t index=(int32_t)(state>>1); /* UTF-8 index */ |
| state&=1; /* 1 if in surrogate pair, must be index>=4 */ |
| |
| if((state==0 ? index<0 : index<4) || iter->limit<index) { |
| *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; |
| } else { |
| iter->start=index; /* restore UTF-8 byte index */ |
| if(index<=1) { |
| iter->index=index; |
| } else { |
| iter->index=-1; /* unknown UTF-16 index */ |
| } |
| if(state==0) { |
| iter->reservedField=0; |
| } else { |
| /* verified index>=4 above */ |
| UChar32 c; |
| L8_PREV((const uint8_t *)iter->context, 0, index, c); |
| if(c<=0xffff) { |
| *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; |
| } else { |
| iter->reservedField=c; |
| } |
| } |
| } |
| } |
| } |
| |
| static const UCharIterator lenient8Iterator={ |
| 0, 0, 0, 0, 0, 0, |
| lenient8IteratorGetIndex, |
| lenient8IteratorMove, |
| lenient8IteratorHasNext, |
| lenient8IteratorHasPrevious, |
| lenient8IteratorCurrent, |
| lenient8IteratorNext, |
| lenient8IteratorPrevious, |
| NULL, |
| lenient8IteratorGetState, |
| lenient8IteratorSetState |
| }; |
| |
| U_CAPI void U_EXPORT2 |
| uiter_setLenient8(UCharIterator *iter, const char *s, int32_t length) { |
| if(iter!=0) { |
| if(s!=0 && length>=-1) { |
| *iter=lenient8Iterator; |
| iter->context=s; |
| if(length>=0) { |
| iter->limit=length; |
| } else { |
| iter->limit=strlen(s); |
| } |
| iter->length= iter->limit<=1 ? iter->limit : -1; |
| } else { |
| /* set no-op iterator */ |
| uiter_setString(iter, NULL, 0); |
| } |
| } |
| } |
