| |
| #***************************************************************************** |
| # |
| # Copyright (C) 2016 and later: Unicode, Inc. and others. |
| # License & terms of use: http://www.unicode.org/copyright.html |
| # |
| #***************************************************************************** |
| #***************************************************************************** |
| # |
| # Copyright (C) 2002-2016, International Business Machines Corporation and others. |
| # All Rights Reserved. |
| # |
| #***************************************************************************** |
| # |
| # file: rbbirpt.txt |
| # ICU Break Iterator Rule Parser State Table |
| # |
| # This state table is used when reading and parsing a set of RBBI rules |
| # The rule parser uses a state machine; the data in this file define the |
| # state transitions that occur for each input character. |
| # |
| # *** This file defines the RBBI rule grammar. This is it. |
| # *** The determination of what is accepted is here. |
| # |
| # This file is processed by a perl script "rbbicst.pl" to produce initialized C arrays |
| # that are then built with the rule parser. |
| # |
| # perl rbbicst.pl < rbbirpt.txt > rbbirpt.h |
| |
| # |
| # Here is the syntax of the state definitions in this file: |
| # |
| # |
| #StateName: |
| # input-char n next-state ^push-state action |
| # input-char n next-state ^push-state action |
| # | | | | | |
| # | | | | |--- action to be performed by state machine |
| # | | | | See function RBBIRuleScanner::doParseActions() |
| # | | | | |
| # | | | |--- Push this named state onto the state stack. |
| # | | | Later, when next state is specified as "pop", |
| # | | | the pushed state will become the current state. |
| # | | | |
| # | | |--- Transition to this state if the current input character matches the input |
| # | | character or char class in the left hand column. "pop" causes the next |
| # | | state to be popped from the state stack. |
| # | | |
| # | |--- When making the state transition specified on this line, advance to the next |
| # | character from the input only if 'n' appears here. |
| # | |
| # |--- Character or named character classes to test for. If the current character being scanned |
| # matches, peform the actions and go to the state specified on this line. |
| # The input character is tested sequentally, in the order written. The characters and |
| # character classes tested for do not need to be mutually exclusive. The first match wins. |
| # |
| |
| |
| |
| |
| # |
| # start state, scan position is at the beginning of the rules file, or in between two rules. |
| # |
| start: |
| escaped term ^break-rule-end doExprStart |
| white_space n start |
| '^' n start-after-caret ^break-rule-end doNoChain |
| '$' scan-var-name ^assign-or-rule doExprStart |
| '!' n rev-option |
| ';' n start # ignore empty rules. |
| eof exit |
| default term ^break-rule-end doExprStart |
| |
| # |
| # break-rule-end: Returned from doing a break-rule expression. |
| # |
| break-rule-end: |
| ';' n start doEndOfRule |
| white_space n break-rule-end |
| default errorDeath doRuleError |
| |
| # |
| # start of a rule, after having seen a '^' (inhibits rule chain in). |
| # Similar to the main 'start' state in most respects, except |
| # - empty rule is an error. |
| # - A second '^' is an error. |
| # |
| start-after-caret: |
| escaped term doExprStart |
| white_space n start-after-caret |
| '^' errorDeath doRuleError # two '^'s |
| '$' scan-var-name ^term-var-ref doExprStart |
| ';' errorDeath doRuleError # ^ ; |
| eof errorDeath doRuleError |
| default term doExprStart |
| |
| # |
| # ! We've just scanned a '!', indicating either a !!key word flag or a |
| # !Reverse rule. |
| # |
| rev-option: |
| '!' n option-scan1 |
| default reverse-rule ^break-rule-end doReverseDir |
| |
| option-scan1: |
| name_start_char n option-scan2 doOptionStart |
| default errorDeath doRuleError |
| |
| option-scan2: |
| name_char n option-scan2 |
| default option-scan3 doOptionEnd |
| |
| option-scan3: |
| ';' n start |
| white_space n option-scan3 |
| default errorDeath doRuleError |
| |
| |
| reverse-rule: |
| default term ^break-rule-end doExprStart |
| |
| |
| # |
| # term. Eat through a single rule character, or a composite thing, which |
| # could be a parenthesized expression, a variable name, or a Unicode Set. |
| # |
| term: |
| escaped n expr-mod doRuleChar |
| white_space n term |
| rule_char n expr-mod doRuleChar |
| '[' scan-unicode-set ^expr-mod |
| '(' n term ^expr-mod doLParen |
| '$' scan-var-name ^term-var-ref |
| '.' n expr-mod doDotAny |
| default errorDeath doRuleError |
| |
| |
| |
| # |
| # term-var-ref We've just finished scanning a reference to a $variable. |
| # Check that the variable was defined. |
| # The variable name scanning is in common with assignment statements, |
| # so the check can't be done there. |
| term-var-ref: |
| default expr-mod doCheckVarDef |
| |
| |
| # |
| # expr-mod We've just finished scanning a term, now look for the optional |
| # trailing '*', '?', '+' |
| # |
| expr-mod: |
| white_space n expr-mod |
| '*' n expr-cont doUnaryOpStar |
| '+' n expr-cont doUnaryOpPlus |
| '?' n expr-cont doUnaryOpQuestion |
| default expr-cont |
| |
| |
| # |
| # expr-cont Expression, continuation. At a point where additional terms are |
| # allowed, but not required. |
| # |
| expr-cont: |
| escaped term doExprCatOperator |
| white_space n expr-cont |
| rule_char term doExprCatOperator |
| '[' term doExprCatOperator |
| '(' term doExprCatOperator |
| '$' term doExprCatOperator |
| '.' term doExprCatOperator |
| '/' look-ahead doExprCatOperator |
| '{' n tag-open doExprCatOperator |
| '|' n term doExprOrOperator |
| ')' n pop doExprRParen |
| default pop doExprFinished |
| |
| |
| # |
| # look-ahead Scanning a '/', which identifies a break point, assuming that the |
| # remainder of the expression matches. |
| # |
| # Generate a parse tree as if this was a special kind of input symbol |
| # appearing in an otherwise normal concatenation expression. |
| # |
| look-ahead: |
| '/' n expr-cont-no-slash doSlash |
| default errorDeath |
| |
| |
| # |
| # expr-cont-no-slash Expression, continuation. At a point where additional terms are |
| # allowed, but not required. Just like |
| # expr-cont, above, except that no '/' |
| # look-ahead symbol is permitted. |
| # |
| expr-cont-no-slash: |
| escaped term doExprCatOperator |
| white_space n expr-cont |
| rule_char term doExprCatOperator |
| '[' term doExprCatOperator |
| '(' term doExprCatOperator |
| '$' term doExprCatOperator |
| '.' term doExprCatOperator |
| '|' n term doExprOrOperator |
| ')' n pop doExprRParen |
| default pop doExprFinished |
| |
| |
| # |
| # tags scanning a '{', the opening delimiter for a tag that identifies |
| # the kind of match. Scan the whole {dddd} tag, where d=digit |
| # |
| tag-open: |
| white_space n tag-open |
| digit_char tag-value doStartTagValue |
| default errorDeath doTagExpectedError |
| |
| tag-value: |
| white_space n tag-close |
| '}' tag-close |
| digit_char n tag-value doTagDigit |
| default errorDeath doTagExpectedError |
| |
| tag-close: |
| white_space n tag-close |
| '}' n expr-cont-no-tag doTagValue |
| default errorDeath doTagExpectedError |
| |
| |
| |
| # |
| # expr-cont-no-tag Expression, continuation. At a point where additional terms are |
| # allowed, but not required. Just like |
| # expr-cont, above, except that no "{ddd}" |
| # tagging is permitted. |
| # |
| expr-cont-no-tag: |
| escaped term doExprCatOperator |
| white_space n expr-cont-no-tag |
| rule_char term doExprCatOperator |
| '[' term doExprCatOperator |
| '(' term doExprCatOperator |
| '$' term doExprCatOperator |
| '.' term doExprCatOperator |
| '/' look-ahead doExprCatOperator |
| '|' n term doExprOrOperator |
| ')' n pop doExprRParen |
| default pop doExprFinished |
| |
| |
| |
| |
| # |
| # Variable Name Scanning. |
| # |
| # The state that branched to here must have pushed a return state |
| # to go to after completion of the variable name scanning. |
| # |
| # The current input character must be the $ that introduces the name. |
| # The $ is consummed here rather than in the state that first detected it |
| # so that the doStartVariableName action only needs to happen in one |
| # place (here), and the other states don't need to worry about it. |
| # |
| scan-var-name: |
| '$' n scan-var-start doStartVariableName |
| default errorDeath |
| |
| |
| scan-var-start: |
| name_start_char n scan-var-body |
| default errorDeath doVariableNameExpectedErr |
| |
| scan-var-body: |
| name_char n scan-var-body |
| default pop doEndVariableName |
| |
| |
| |
| # |
| # scan-unicode-set Unicode Sets are parsed by the the UnicodeSet class. |
| # Within the RBBI parser, after finding the first character |
| # of a Unicode Set, we just hand the rule input at that |
| # point of to the Unicode Set constructor, then pick |
| # up parsing after the close of the set. |
| # |
| # The action for this state invokes the UnicodeSet parser. |
| # |
| scan-unicode-set: |
| '[' n pop doScanUnicodeSet |
| 'p' n pop doScanUnicodeSet |
| 'P' n pop doScanUnicodeSet |
| default errorDeath |
| |
| |
| |
| |
| |
| |
| |
| # |
| # assign-or-rule. A $variable was encountered at the start of something, could be |
| # either an assignment statement or a rule, depending on whether an '=' |
| # follows the variable name. We get to this state when the variable name |
| # scanning does a return. |
| # |
| assign-or-rule: |
| white_space n assign-or-rule |
| '=' n term ^assign-end doStartAssign # variable was target of assignment |
| default term-var-ref ^break-rule-end # variable was a term in a rule |
| |
| |
| |
| # |
| # assign-end This state is entered when the end of the expression on the |
| # right hand side of an assignment is found. We get here via |
| # a pop; this state is pushed when the '=' in an assignment is found. |
| # |
| # The only thing allowed at this point is a ';'. The RHS of an |
| # assignment must look like a rule expression, and we come here |
| # when what is being scanned no longer looks like an expression. |
| # |
| assign-end: |
| ';' n start doEndAssign |
| default errorDeath doRuleErrorAssignExpr |
| |
| |
| |
| # |
| # errorDeath. This state is specified as the next state whenever a syntax error |
| # in the source rules is detected. Barring bugs, the state machine will never |
| # actually get here, but will stop because of the action associated with the error. |
| # But, just in case, this state asks the state machine to exit. |
| errorDeath: |
| default n errorDeath doExit |
| |
| |
