| #!/usr/bin/perl |
| # ******************************************************************** |
| # * COPYRIGHT: |
| # * Copyright (c) 2002-2015, International Business Machines Corporation and |
| # * others. All Rights Reserved. |
| # ******************************************************************** |
| # |
| # regexcst.pl |
| # Compile the regular expression paser state table data into initialized C data. |
| # Usage: |
| # cd icu/source/i18n |
| # perl regexcst.pl < regexcst.txt > regexcst.h |
| # |
| # The output file, regexcst.h, is included by some of the .cpp regex |
| # implementation files. This perl script is NOT run as part |
| # of a normal ICU build. It is run by hand when needed, and the |
| # regexcst.h generated file is put back into cvs. |
| # |
| # See regexcst.txt for a description of the input format for this script. |
| # |
| # This script is derived from rbbicst.pl, which peforms the same function |
| # for the Rule Based Break Iterator Rule Parser. Perhaps they could be |
| # merged? |
| # |
| |
| |
| $num_states = 1; # Always the state number for the line being compiled. |
| $line_num = 0; # The line number in the input file. |
| |
| $states{"pop"} = 255; # Add the "pop" to the list of defined state names. |
| # This prevents any state from being labelled with "pop", |
| # and resolves references to "pop" in the next state field. |
| |
| line_loop: while (<>) { |
| chomp(); |
| $line = $_; |
| @fields = split(); |
| $line_num++; |
| |
| # Remove # comments, which are any fields beginning with a #, plus all |
| # that follow on the line. |
| for ($i=0; $i<@fields; $i++) { |
| if ($fields[$i] =~ /^#/) { |
| @fields = @fields[0 .. $i-1]; |
| last; |
| } |
| } |
| # ignore blank lines, and those with no fields left after stripping comments.. |
| if (@fields == 0) { |
| next; |
| } |
| |
| # |
| # State Label: handling. |
| # Does the first token end with a ":"? If so, it's the name of a state. |
| # Put in a hash, together with the current state number, |
| # so that we can later look up the number from the name. |
| # |
| if (@fields[0] =~ /.*:$/) { |
| $state_name = @fields[0]; |
| $state_name =~ s/://; # strip off the colon from the state name. |
| |
| if ($states{$state_name} != 0) { |
| print " rbbicst: at line $line-num duplicate definition of state $state_name\n"; |
| } |
| $states{$state_name} = $num_states; |
| $stateNames[$num_states] = $state_name; |
| |
| # if the label was the only thing on this line, go on to the next line, |
| # otherwise assume that a state definition is on the same line and fall through. |
| if (@fields == 1) { |
| next line_loop; |
| } |
| shift @fields; # shift off label field in preparation |
| # for handling the rest of the line. |
| } |
| |
| # |
| # State Transition line. |
| # syntax is this, |
| # character [n] target-state [^push-state] [function-name] |
| # where |
| # [something] is an optional something |
| # character is either a single quoted character e.g. '[' |
| # or a name of a character class, e.g. white_space |
| # |
| |
| $state_line_num[$num_states] = $line_num; # remember line number with each state |
| # so we can make better error messages later. |
| # |
| # First field, character class or literal character for this transition. |
| # |
| if ($fields[0] =~ /^'.'$/) { |
| # We've got a quoted literal character. |
| $state_literal_chars[$num_states] = $fields[0]; |
| $state_literal_chars[$num_states] =~ s/'//g; |
| } else { |
| # We've got the name of a character class. |
| $state_char_class[$num_states] = $fields[0]; |
| if ($fields[0] =~ /[\W]/) { |
| print " rbbicsts: at line $line_num, bad character literal or character class name.\n"; |
| print " scanning $fields[0]\n"; |
| exit(-1); |
| } |
| } |
| shift @fields; |
| |
| # |
| # do the 'n' flag |
| # |
| $state_flag[$num_states] = "FALSE"; |
| if ($fields[0] eq "n") { |
| $state_flag[$num_states] = "TRUE"; |
| shift @fields; |
| } |
| |
| # |
| # do the destination state. |
| # |
| $state_dest_state[$num_states] = $fields[0]; |
| if ($fields[0] eq "") { |
| print " rbbicsts: at line $line_num, destination state missing.\n"; |
| exit(-1); |
| } |
| shift @fields; |
| |
| # |
| # do the push state, if present. |
| # |
| if ($fields[0] =~ /^\^/) { |
| $fields[0] =~ s/^\^//; |
| $state_push_state[$num_states] = $fields[0]; |
| if ($fields[0] eq "" ) { |
| print " rbbicsts: at line $line_num, expected state after ^ (no spaces).\n"; |
| exit(-1); |
| } |
| shift @fields; |
| } |
| |
| # |
| # Lastly, do the optional action name. |
| # |
| if ($fields[0] ne "") { |
| $state_func_name[$num_states] = $fields[0]; |
| shift @fields; |
| } |
| |
| # |
| # There should be no fields left on the line at this point. |
| # |
| if (@fields > 0) { |
| print " rbbicsts: at line $line_num, unexpected extra stuff on input line.\n"; |
| print " scanning $fields[0]\n"; |
| } |
| $num_states++; |
| } |
| |
| # |
| # We've read in the whole file, now go back and output the |
| # C source code for the state transition table. |
| # |
| # We read all states first, before writing anything, so that the state numbers |
| # for the destination states are all available to be written. |
| # |
| |
| # |
| # Make hashes for the names of the character classes and |
| # for the names of the actions that appeared. |
| # |
| for ($state=1; $state < $num_states; $state++) { |
| if ($state_char_class[$state] ne "") { |
| if ($charClasses{$state_char_class[$state]} == 0) { |
| $charClasses{$state_char_class[$state]} = 1; |
| } |
| } |
| if ($state_func_name[$state] eq "") { |
| $state_func_name[$state] = "doNOP"; |
| } |
| if ($actions{$state_action_name[$state]} == 0) { |
| $actions{$state_func_name[$state]} = 1; |
| } |
| } |
| |
| # |
| # Check that all of the destination states have been defined |
| # |
| # |
| $states{"exit"} = 0; # Predefined state name, terminates state machine. |
| for ($state=1; $state<$num_states; $state++) { |
| if ($states{$state_dest_state[$state]} == 0 && $state_dest_state[$state] ne "exit") { |
| print "Error at line $state_line_num[$state]: target state \"$state_dest_state[$state]\" is not defined.\n"; |
| $errors++; |
| } |
| if ($state_push_state[$state] ne "" && $states{$state_push_state[$state]} == 0) { |
| print "Error at line $state_line_num[$state]: target state \"$state_push_state[$state]\" is not defined.\n"; |
| $errors++; |
| } |
| } |
| |
| die if ($errors>0); |
| |
| print "//---------------------------------------------------------------------------------\n"; |
| print "//\n"; |
| print "// Generated Header File. Do not edit by hand.\n"; |
| print "// This file contains the state table for the ICU Regular Expression Pattern Parser\n"; |
| print "// It is generated by the Perl script \"regexcst.pl\" from\n"; |
| print "// the rule parser state definitions file \"regexcst.txt\".\n"; |
| print "//\n"; |
| print "// Copyright (C) 2002-2015 International Business Machines Corporation \n"; |
| print "// and others. All rights reserved. \n"; |
| print "//\n"; |
| print "//---------------------------------------------------------------------------------\n"; |
| print "#ifndef RBBIRPT_H\n"; |
| print "#define RBBIRPT_H\n"; |
| print "\n"; |
| print "U_NAMESPACE_BEGIN\n"; |
| |
| # |
| # Emit the constants for indicies of Unicode Sets |
| # Define one constant for each of the character classes encountered. |
| # At the same time, store the index corresponding to the set name back into hash. |
| # |
| print "//\n"; |
| print "// Character classes for regex pattern scanning.\n"; |
| print "//\n"; |
| $i = 128; # State Table values for Unicode char sets range from 128-250. |
| # Sets "default", "quoted", etc. get special handling. |
| # They have no corresponding UnicodeSet object in the state machine, |
| # but are handled by special case code. So we emit no reference |
| # to a UnicodeSet object to them here. |
| foreach $setName (keys %charClasses) { |
| if ($setName eq "default") { |
| $charClasses{$setName} = 255;} |
| elsif ($setName eq "quoted") { |
| $charClasses{$setName} = 254;} |
| elsif ($setName eq "eof") { |
| $charClasses{$setName} = 253;} |
| else { |
| # Normal character class. Fill in array with a ptr to the corresponding UnicodeSet in the state machine. |
| print " static const uint8_t kRuleSet_$setName = $i;\n"; |
| $charClasses{$setName} = $i; |
| $i++; |
| } |
| } |
| print "\n\n"; |
| |
| # |
| # Emit the enum for the actions to be performed. |
| # |
| print "enum Regex_PatternParseAction {\n"; |
| foreach $act (keys %actions) { |
| print " $act,\n"; |
| } |
| print " rbbiLastAction};\n\n"; |
| |
| # |
| # Emit the struct definition for transtion table elements. |
| # |
| print "//-------------------------------------------------------------------------------\n"; |
| print "//\n"; |
| print "// RegexTableEl represents the structure of a row in the transition table\n"; |
| print "// for the pattern parser state machine.\n"; |
| print "//-------------------------------------------------------------------------------\n"; |
| print "struct RegexTableEl {\n"; |
| print " Regex_PatternParseAction fAction;\n"; |
| print " uint8_t fCharClass; // 0-127: an individual ASCII character\n"; |
| print " // 128-255: character class index\n"; |
| print " uint8_t fNextState; // 0-250: normal next-state numbers\n"; |
| print " // 255: pop next-state from stack.\n"; |
| print " uint8_t fPushState;\n"; |
| print " UBool fNextChar;\n"; |
| print "};\n\n"; |
| |
| # |
| # emit the state transition table |
| # |
| print "static const struct RegexTableEl gRuleParseStateTable[] = {\n"; |
| print " {doNOP, 0, 0, 0, TRUE}\n"; # State 0 is a dummy. Real states start with index = 1. |
| for ($state=1; $state < $num_states; $state++) { |
| print " , {$state_func_name[$state],"; |
| if ($state_literal_chars[$state] ne "") { |
| $c = $state_literal_chars[$state]; |
| printf(" %d /* $c */,", ord($c)); # use numeric value, so EBCDIC machines are ok. |
| }else { |
| print " $charClasses{$state_char_class[$state]},"; |
| } |
| print " $states{$state_dest_state[$state]},"; |
| |
| # The push-state field is optional. If omitted, fill field with a zero, which flags |
| # the state machine that there is no push state. |
| if ($state_push_state[$state] eq "") { |
| print "0, "; |
| } else { |
| print " $states{$state_push_state[$state]},"; |
| } |
| print " $state_flag[$state]} "; |
| |
| # Put out a C++ comment showing the number (index) of this state row, |
| # and, if this is the first row of the table for this state, the state name. |
| print " // $state "; |
| if ($stateNames[$state] ne "") { |
| print " $stateNames[$state]"; |
| } |
| print "\n"; |
| }; |
| print " };\n"; |
| |
| |
| # |
| # emit a mapping array from state numbers to state names. |
| # |
| # This array is used for producing debugging output from the pattern parser. |
| # |
| print "static const char * const RegexStateNames[] = {"; |
| for ($state=0; $state<$num_states; $state++) { |
| if ($stateNames[$state] ne "") { |
| print " \"$stateNames[$state]\",\n"; |
| } else { |
| print " 0,\n"; |
| } |
| } |
| print " 0};\n\n"; |
| |
| print "U_NAMESPACE_END\n"; |
| print "#endif\n"; |
| |
| |
| |
