| // © 2016 and later: Unicode, Inc. and others. | |
| // License & terms of use: http://www.unicode.org/copyright.html | |
| /* | |
| ******************************************************************************* | |
| * Copyright (C) 2011-2014, International Business Machines | |
| * Corporation and others. All Rights Reserved. | |
| ******************************************************************************* | |
| * file name: ppucd.cpp | |
| * encoding: UTF-8 | |
| * tab size: 8 (not used) | |
| * indentation:4 | |
| * | |
| * created on: 2011dec11 | |
| * created by: Markus W. Scherer | |
| */ | |
| #include "unicode/utypes.h" | |
| #include "unicode/uchar.h" | |
| #include "charstr.h" | |
| #include "cstring.h" | |
| #include "ppucd.h" | |
| #include "uassert.h" | |
| #include "uparse.h" | |
| #include <stdio.h> | |
| #include <string.h> | |
| U_NAMESPACE_BEGIN | |
| PropertyNames::~PropertyNames() {} | |
| int32_t | |
| PropertyNames::getPropertyEnum(const char *name) const { | |
| return u_getPropertyEnum(name); | |
| } | |
| int32_t | |
| PropertyNames::getPropertyValueEnum(int32_t property, const char *name) const { | |
| return u_getPropertyValueEnum((UProperty)property, name); | |
| } | |
| UniProps::UniProps() | |
| : start(U_SENTINEL), end(U_SENTINEL), | |
| bmg(U_SENTINEL), bpb(U_SENTINEL), | |
| scf(U_SENTINEL), slc(U_SENTINEL), stc(U_SENTINEL), suc(U_SENTINEL), | |
| digitValue(-1), numericValue(NULL), | |
| name(NULL), nameAlias(NULL) { | |
| memset(binProps, 0, sizeof(binProps)); | |
| memset(intProps, 0, sizeof(intProps)); | |
| memset(age, 0, 4); | |
| } | |
| UniProps::~UniProps() {} | |
| const int32_t PreparsedUCD::kNumLineBuffers; | |
| PreparsedUCD::PreparsedUCD(const char *filename, UErrorCode &errorCode) | |
| : icuPnames(new PropertyNames()), pnames(icuPnames), | |
| file(NULL), | |
| defaultLineIndex(-1), blockLineIndex(-1), lineIndex(0), | |
| lineNumber(0), | |
| lineType(NO_LINE), | |
| fieldLimit(NULL), lineLimit(NULL) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| if(filename==NULL || *filename==0 || (*filename=='-' && filename[1]==0)) { | |
| filename=NULL; | |
| file=stdin; | |
| } else { | |
| file=fopen(filename, "r"); | |
| } | |
| if(file==NULL) { | |
| perror("error opening preparsed UCD"); | |
| fprintf(stderr, "error opening preparsed UCD file %s\n", filename ? filename : "\"no file name given\""); | |
| errorCode=U_FILE_ACCESS_ERROR; | |
| return; | |
| } | |
| memset(ucdVersion, 0, 4); | |
| lines[0][0]=0; | |
| } | |
| PreparsedUCD::~PreparsedUCD() { | |
| if(file!=stdin) { | |
| fclose(file); | |
| } | |
| delete icuPnames; | |
| } | |
| // Same order as the LineType values. | |
| static const char *lineTypeStrings[]={ | |
| NULL, | |
| NULL, | |
| "ucd", | |
| "property", | |
| "binary", | |
| "value", | |
| "defaults", | |
| "block", | |
| "cp", | |
| "unassigned", | |
| "algnamesrange" | |
| }; | |
| PreparsedUCD::LineType | |
| PreparsedUCD::readLine(UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return NO_LINE; } | |
| // Select the next available line buffer. | |
| while(!isLineBufferAvailable(lineIndex)) { | |
| ++lineIndex; | |
| if (lineIndex == kNumLineBuffers) { | |
| lineIndex = 0; | |
| } | |
| } | |
| char *line=lines[lineIndex]; | |
| *line=0; | |
| lineLimit=fieldLimit=line; | |
| lineType=NO_LINE; | |
| char *result=fgets(line, sizeof(lines[0]), file); | |
| if(result==NULL) { | |
| if(ferror(file)) { | |
| perror("error reading preparsed UCD"); | |
| fprintf(stderr, "error reading preparsed UCD before line %ld\n", (long)lineNumber); | |
| errorCode=U_FILE_ACCESS_ERROR; | |
| } | |
| return NO_LINE; | |
| } | |
| ++lineNumber; | |
| if(*line=='#') { | |
| fieldLimit=strchr(line, 0); | |
| return lineType=EMPTY_LINE; | |
| } | |
| // Remove trailing /r/n. | |
| char c; | |
| char *limit=strchr(line, 0); | |
| while(line<limit && ((c=*(limit-1))=='\n' || c=='\r')) { --limit; } | |
| // Remove trailing white space. | |
| while(line<limit && ((c=*(limit-1))==' ' || c=='\t')) { --limit; } | |
| *limit=0; | |
| lineLimit=limit; | |
| if(line==limit) { | |
| fieldLimit=limit; | |
| return lineType=EMPTY_LINE; | |
| } | |
| // Split by ';'. | |
| char *semi=line; | |
| while((semi=strchr(semi, ';'))!=NULL) { *semi++=0; } | |
| fieldLimit=strchr(line, 0); | |
| // Determine the line type. | |
| int32_t type; | |
| for(type=EMPTY_LINE+1;; ++type) { | |
| if(type==LINE_TYPE_COUNT) { | |
| fprintf(stderr, | |
| "error in preparsed UCD: unknown line type (first field) '%s' on line %ld\n", | |
| line, (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| return NO_LINE; | |
| } | |
| if(0==strcmp(line, lineTypeStrings[type])) { | |
| break; | |
| } | |
| } | |
| lineType=(LineType)type; | |
| if(lineType==UNICODE_VERSION_LINE && fieldLimit<lineLimit) { | |
| u_versionFromString(ucdVersion, fieldLimit+1); | |
| } | |
| return lineType; | |
| } | |
| const char * | |
| PreparsedUCD::firstField() { | |
| char *field=lines[lineIndex]; | |
| fieldLimit=strchr(field, 0); | |
| return field; | |
| } | |
| const char * | |
| PreparsedUCD::nextField() { | |
| if(fieldLimit==lineLimit) { return NULL; } | |
| char *field=fieldLimit+1; | |
| fieldLimit=strchr(field, 0); | |
| return field; | |
| } | |
| const UniProps * | |
| PreparsedUCD::getProps(UnicodeSet &newValues, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return NULL; } | |
| newValues.clear(); | |
| if(!lineHasPropertyValues()) { | |
| errorCode=U_ILLEGAL_ARGUMENT_ERROR; | |
| return NULL; | |
| } | |
| firstField(); | |
| const char *field=nextField(); | |
| if(field==NULL) { | |
| // No range field after the type. | |
| fprintf(stderr, | |
| "error in preparsed UCD: missing default/block/cp range field " | |
| "(no second field) on line %ld\n", | |
| (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| return NULL; | |
| } | |
| UChar32 start, end; | |
| if(!parseCodePointRange(field, start, end, errorCode)) { return NULL; } | |
| UniProps *props; | |
| UBool insideBlock=FALSE; // TRUE if cp or unassigned range inside the block range. | |
| switch(lineType) { | |
| case DEFAULTS_LINE: | |
| // Should occur before any block/cp/unassigned line. | |
| if(blockLineIndex>=0) { | |
| fprintf(stderr, | |
| "error in preparsed UCD: default line %ld after one or more block lines\n", | |
| (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| return NULL; | |
| } | |
| if(defaultLineIndex>=0) { | |
| fprintf(stderr, | |
| "error in preparsed UCD: second line with default properties on line %ld\n", | |
| (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| return NULL; | |
| } | |
| if(start!=0 || end!=0x10ffff) { | |
| fprintf(stderr, | |
| "error in preparsed UCD: default range must be 0..10FFFF, not '%s' on line %ld\n", | |
| field, (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| return NULL; | |
| } | |
| props=&defaultProps; | |
| defaultLineIndex=lineIndex; | |
| break; | |
| case BLOCK_LINE: | |
| blockProps=defaultProps; // Block inherits default properties. | |
| props=&blockProps; | |
| blockLineIndex=lineIndex; | |
| break; | |
| case CP_LINE: | |
| case UNASSIGNED_LINE: | |
| if(blockProps.start<=start && end<=blockProps.end) { | |
| insideBlock=TRUE; | |
| if(lineType==CP_LINE) { | |
| // Code point range fully inside the last block inherits the block properties. | |
| cpProps=blockProps; | |
| } else { | |
| // Unassigned line inside the block is based on default properties | |
| // which override block properties. | |
| cpProps=defaultProps; | |
| newValues=blockValues; | |
| // Except, it inherits the one blk=Block property. | |
| int32_t blkIndex=UCHAR_BLOCK-UCHAR_INT_START; | |
| cpProps.intProps[blkIndex]=blockProps.intProps[blkIndex]; | |
| newValues.remove((UChar32)UCHAR_BLOCK); | |
| } | |
| } else if(start>blockProps.end || end<blockProps.start) { | |
| // Code point range fully outside the last block inherits the default properties. | |
| cpProps=defaultProps; | |
| } else { | |
| // Code point range partially overlapping with the last block is illegal. | |
| fprintf(stderr, | |
| "error in preparsed UCD: cp range %s on line %ld only " | |
| "partially overlaps with block range %04lX..%04lX\n", | |
| field, (long)lineNumber, (long)blockProps.start, (long)blockProps.end); | |
| errorCode=U_PARSE_ERROR; | |
| return NULL; | |
| } | |
| props=&cpProps; | |
| break; | |
| default: | |
| // Will not occur because of the range check above. | |
| errorCode=U_ILLEGAL_ARGUMENT_ERROR; | |
| return NULL; | |
| } | |
| props->start=start; | |
| props->end=end; | |
| while((field=nextField())!=NULL) { | |
| if(!parseProperty(*props, field, newValues, errorCode)) { return NULL; } | |
| } | |
| if(lineType==BLOCK_LINE) { | |
| blockValues=newValues; | |
| } else if(lineType==UNASSIGNED_LINE && insideBlock) { | |
| // Unset newValues for values that are the same as the block values. | |
| for(int32_t prop=0; prop<UCHAR_BINARY_LIMIT; ++prop) { | |
| if(newValues.contains(prop) && cpProps.binProps[prop]==blockProps.binProps[prop]) { | |
| newValues.remove(prop); | |
| } | |
| } | |
| for(int32_t prop=UCHAR_INT_START; prop<UCHAR_INT_LIMIT; ++prop) { | |
| int32_t index=prop-UCHAR_INT_START; | |
| if(newValues.contains(prop) && cpProps.intProps[index]==blockProps.intProps[index]) { | |
| newValues.remove(prop); | |
| } | |
| } | |
| } | |
| return props; | |
| } | |
| static const struct { | |
| const char *name; | |
| int32_t prop; | |
| } ppucdProperties[]={ | |
| { "Name_Alias", PPUCD_NAME_ALIAS }, | |
| { "Conditional_Case_Mappings", PPUCD_CONDITIONAL_CASE_MAPPINGS }, | |
| { "Turkic_Case_Folding", PPUCD_TURKIC_CASE_FOLDING } | |
| }; | |
| // Returns TRUE for "ok to continue parsing fields". | |
| UBool | |
| PreparsedUCD::parseProperty(UniProps &props, const char *field, UnicodeSet &newValues, | |
| UErrorCode &errorCode) { | |
| CharString pBuffer; | |
| const char *p=field; | |
| const char *v=strchr(p, '='); | |
| int binaryValue; | |
| if(*p=='-') { | |
| if(v!=NULL) { | |
| fprintf(stderr, | |
| "error in preparsed UCD: mix of binary-property-no and " | |
| "enum-property syntax '%s' on line %ld\n", | |
| field, (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| return FALSE; | |
| } | |
| binaryValue=0; | |
| ++p; | |
| } else if(v==NULL) { | |
| binaryValue=1; | |
| } else { | |
| binaryValue=-1; | |
| // Copy out the property name rather than modifying the field (writing a NUL). | |
| pBuffer.append(p, (int32_t)(v-p), errorCode); | |
| p=pBuffer.data(); | |
| ++v; | |
| } | |
| int32_t prop=pnames->getPropertyEnum(p); | |
| if(prop<0) { | |
| for(int32_t i=0;; ++i) { | |
| if(i==UPRV_LENGTHOF(ppucdProperties)) { | |
| // Ignore unknown property names. | |
| return TRUE; | |
| } | |
| if(0==uprv_stricmp(p, ppucdProperties[i].name)) { | |
| prop=ppucdProperties[i].prop; | |
| U_ASSERT(prop>=0); | |
| break; | |
| } | |
| } | |
| } | |
| if(prop<UCHAR_BINARY_LIMIT) { | |
| if(binaryValue>=0) { | |
| props.binProps[prop]=(UBool)binaryValue; | |
| } else { | |
| // No binary value for a binary property. | |
| fprintf(stderr, | |
| "error in preparsed UCD: enum-property syntax '%s' " | |
| "for binary property on line %ld\n", | |
| field, (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| } | |
| } else if(binaryValue>=0) { | |
| // Binary value for a non-binary property. | |
| fprintf(stderr, | |
| "error in preparsed UCD: binary-property syntax '%s' " | |
| "for non-binary property on line %ld\n", | |
| field, (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| } else if (prop < UCHAR_INT_START) { | |
| fprintf(stderr, | |
| "error in preparsed UCD: prop value is invalid: '%d' for line %ld\n", | |
| prop, (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| } else if(prop<UCHAR_INT_LIMIT) { | |
| int32_t value=pnames->getPropertyValueEnum(prop, v); | |
| if(value==UCHAR_INVALID_CODE && prop==UCHAR_CANONICAL_COMBINING_CLASS) { | |
| // TODO: Make getPropertyValueEnum(UCHAR_CANONICAL_COMBINING_CLASS, v) work. | |
| char *end; | |
| unsigned long ccc=uprv_strtoul(v, &end, 10); | |
| if(v<end && *end==0 && ccc<=254) { | |
| value=(int32_t)ccc; | |
| } | |
| } | |
| if(value==UCHAR_INVALID_CODE) { | |
| fprintf(stderr, | |
| "error in preparsed UCD: '%s' is not a valid value on line %ld\n", | |
| field, (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| } else { | |
| props.intProps[prop-UCHAR_INT_START]=value; | |
| } | |
| } else if(*v=='<') { | |
| // Do not parse default values like <code point>, just set null values. | |
| switch(prop) { | |
| case UCHAR_BIDI_MIRRORING_GLYPH: | |
| props.bmg=U_SENTINEL; | |
| break; | |
| case UCHAR_BIDI_PAIRED_BRACKET: | |
| props.bpb=U_SENTINEL; | |
| break; | |
| case UCHAR_SIMPLE_CASE_FOLDING: | |
| props.scf=U_SENTINEL; | |
| break; | |
| case UCHAR_SIMPLE_LOWERCASE_MAPPING: | |
| props.slc=U_SENTINEL; | |
| break; | |
| case UCHAR_SIMPLE_TITLECASE_MAPPING: | |
| props.stc=U_SENTINEL; | |
| break; | |
| case UCHAR_SIMPLE_UPPERCASE_MAPPING: | |
| props.suc=U_SENTINEL; | |
| break; | |
| case UCHAR_CASE_FOLDING: | |
| props.cf.remove(); | |
| break; | |
| case UCHAR_LOWERCASE_MAPPING: | |
| props.lc.remove(); | |
| break; | |
| case UCHAR_TITLECASE_MAPPING: | |
| props.tc.remove(); | |
| break; | |
| case UCHAR_UPPERCASE_MAPPING: | |
| props.uc.remove(); | |
| break; | |
| case UCHAR_SCRIPT_EXTENSIONS: | |
| props.scx.clear(); | |
| break; | |
| default: | |
| fprintf(stderr, | |
| "error in preparsed UCD: '%s' is not a valid default value on line %ld\n", | |
| field, (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| } | |
| } else { | |
| char c; | |
| switch(prop) { | |
| case UCHAR_NUMERIC_VALUE: | |
| props.numericValue=v; | |
| c=*v; | |
| if('0'<=c && c<='9' && v[1]==0) { | |
| props.digitValue=c-'0'; | |
| } else { | |
| props.digitValue=-1; | |
| } | |
| break; | |
| case UCHAR_NAME: | |
| props.name=v; | |
| break; | |
| case UCHAR_AGE: | |
| u_versionFromString(props.age, v); // Writes 0.0.0.0 if v is not numeric. | |
| break; | |
| case UCHAR_BIDI_MIRRORING_GLYPH: | |
| props.bmg=parseCodePoint(v, errorCode); | |
| break; | |
| case UCHAR_BIDI_PAIRED_BRACKET: | |
| props.bpb=parseCodePoint(v, errorCode); | |
| break; | |
| case UCHAR_SIMPLE_CASE_FOLDING: | |
| props.scf=parseCodePoint(v, errorCode); | |
| break; | |
| case UCHAR_SIMPLE_LOWERCASE_MAPPING: | |
| props.slc=parseCodePoint(v, errorCode); | |
| break; | |
| case UCHAR_SIMPLE_TITLECASE_MAPPING: | |
| props.stc=parseCodePoint(v, errorCode); | |
| break; | |
| case UCHAR_SIMPLE_UPPERCASE_MAPPING: | |
| props.suc=parseCodePoint(v, errorCode); | |
| break; | |
| case UCHAR_CASE_FOLDING: | |
| parseString(v, props.cf, errorCode); | |
| break; | |
| case UCHAR_LOWERCASE_MAPPING: | |
| parseString(v, props.lc, errorCode); | |
| break; | |
| case UCHAR_TITLECASE_MAPPING: | |
| parseString(v, props.tc, errorCode); | |
| break; | |
| case UCHAR_UPPERCASE_MAPPING: | |
| parseString(v, props.uc, errorCode); | |
| break; | |
| case PPUCD_NAME_ALIAS: | |
| props.nameAlias=v; | |
| break; | |
| case PPUCD_CONDITIONAL_CASE_MAPPINGS: | |
| case PPUCD_TURKIC_CASE_FOLDING: | |
| // No need to parse their values: They are hardcoded in the runtime library. | |
| break; | |
| case UCHAR_SCRIPT_EXTENSIONS: | |
| parseScriptExtensions(v, props.scx, errorCode); | |
| break; | |
| default: | |
| // Ignore unhandled properties. | |
| return TRUE; | |
| } | |
| } | |
| if(U_SUCCESS(errorCode)) { | |
| newValues.add((UChar32)prop); | |
| return TRUE; | |
| } else { | |
| return FALSE; | |
| } | |
| } | |
| UBool | |
| PreparsedUCD::getRangeForAlgNames(UChar32 &start, UChar32 &end, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return FALSE; } | |
| if(lineType!=ALG_NAMES_RANGE_LINE) { | |
| errorCode=U_ILLEGAL_ARGUMENT_ERROR; | |
| return FALSE; | |
| } | |
| firstField(); | |
| const char *field=nextField(); | |
| if(field==NULL) { | |
| // No range field after the type. | |
| fprintf(stderr, | |
| "error in preparsed UCD: missing algnamesrange range field " | |
| "(no second field) on line %ld\n", | |
| (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| return FALSE; | |
| } | |
| return parseCodePointRange(field, start, end, errorCode); | |
| } | |
| UChar32 | |
| PreparsedUCD::parseCodePoint(const char *s, UErrorCode &errorCode) { | |
| char *end; | |
| uint32_t value=(uint32_t)uprv_strtoul(s, &end, 16); | |
| if(end<=s || *end!=0 || value>=0x110000) { | |
| fprintf(stderr, | |
| "error in preparsed UCD: '%s' is not a valid code point on line %ld\n", | |
| s, (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| return U_SENTINEL; | |
| } | |
| return (UChar32)value; | |
| } | |
| UBool | |
| PreparsedUCD::parseCodePointRange(const char *s, UChar32 &start, UChar32 &end, UErrorCode &errorCode) { | |
| uint32_t st, e; | |
| u_parseCodePointRange(s, &st, &e, &errorCode); | |
| if(U_FAILURE(errorCode)) { | |
| fprintf(stderr, | |
| "error in preparsed UCD: '%s' is not a valid code point range on line %ld\n", | |
| s, (long)lineNumber); | |
| return FALSE; | |
| } | |
| start=(UChar32)st; | |
| end=(UChar32)e; | |
| return TRUE; | |
| } | |
| void | |
| PreparsedUCD::parseString(const char *s, UnicodeString &uni, UErrorCode &errorCode) { | |
| UChar *buffer=toUCharPtr(uni.getBuffer(-1)); | |
| int32_t length=u_parseString(s, buffer, uni.getCapacity(), NULL, &errorCode); | |
| if(errorCode==U_BUFFER_OVERFLOW_ERROR) { | |
| errorCode=U_ZERO_ERROR; | |
| uni.releaseBuffer(0); | |
| buffer=toUCharPtr(uni.getBuffer(length)); | |
| length=u_parseString(s, buffer, uni.getCapacity(), NULL, &errorCode); | |
| } | |
| uni.releaseBuffer(length); | |
| if(U_FAILURE(errorCode)) { | |
| fprintf(stderr, | |
| "error in preparsed UCD: '%s' is not a valid Unicode string on line %ld\n", | |
| s, (long)lineNumber); | |
| } | |
| } | |
| void | |
| PreparsedUCD::parseScriptExtensions(const char *s, UnicodeSet &scx, UErrorCode &errorCode) { | |
| if(U_FAILURE(errorCode)) { return; } | |
| scx.clear(); | |
| CharString scString; | |
| for(;;) { | |
| const char *scs; | |
| const char *scLimit=strchr(s, ' '); | |
| if(scLimit!=NULL) { | |
| scs=scString.clear().append(s, (int32_t)(scLimit-s), errorCode).data(); | |
| if(U_FAILURE(errorCode)) { return; } | |
| } else { | |
| scs=s; | |
| } | |
| int32_t script=pnames->getPropertyValueEnum(UCHAR_SCRIPT, scs); | |
| if(script==UCHAR_INVALID_CODE) { | |
| fprintf(stderr, | |
| "error in preparsed UCD: '%s' is not a valid script code on line %ld\n", | |
| scs, (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| return; | |
| } else if(scx.contains(script)) { | |
| fprintf(stderr, | |
| "error in preparsed UCD: scx has duplicate '%s' codes on line %ld\n", | |
| scs, (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| return; | |
| } else { | |
| scx.add(script); | |
| } | |
| if(scLimit!=NULL) { | |
| s=scLimit+1; | |
| } else { | |
| break; | |
| } | |
| } | |
| if(scx.isEmpty()) { | |
| fprintf(stderr, "error in preparsed UCD: empty scx= on line %ld\n", (long)lineNumber); | |
| errorCode=U_PARSE_ERROR; | |
| } | |
| } | |
| U_NAMESPACE_END |