5079cb165264b9459719e46d333421dcc4d79100
[idea/community.git] / tools / lexer / jflex-1.4 / src / JFlex / LexScan.flex
1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2  * JFlex 1.4.3                                                             *
3  * Copyright (C) 1998-2009  Gerwin Klein <lsf@jflex.de>                    *
4  * All rights reserved.                                                    *
5  *                                                                         *
6  * This program is free software; you can redistribute it and/or modify    *
7  * it under the terms of the GNU General Public License. See the file      *
8  * COPYRIGHT for more information.                                         *
9  *                                                                         *
10  * This program is distributed in the hope that it will be useful,         *
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
13  * GNU General Public License for more details.                            *
14  *                                                                         *
15  * You should have received a copy of the GNU General Public License along *
16  * with this program; if not, write to the Free Software Foundation, Inc., *
17  * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
18  *                                                                         *
19  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
20
21 package JFlex;
22
23 import java_cup.runtime.Symbol;
24 import java.util.Vector;
25 import java.io.*;
26 import java.util.Stack;
27
28 /**
29  * The lexer of JFlex.
30  *
31  * Generated by <a href="http://www.jflex.de/">JFlex</a>.
32  *
33  * @author Gerwin Klein
34  * @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
35  */
36 %%
37
38 %final
39 %public
40 %class LexScan
41 %implements sym, java_cup.runtime.Scanner
42 %function next_token
43
44 %type Symbol
45 %unicode
46
47 %column
48 %line
49
50 %eofclose
51
52 %state COMMENT, STATELIST, MACROS, REGEXPSTART
53 %state REGEXP, JAVA_CODE, STATES, STRING_CONTENT
54 %state CHARCLASS, COPY, REPEATEXP, EATWSPNL
55 %state CTOR_ARG
56
57 %cupdebug
58
59 %{  
60   int balance = 0;
61   int commentbalance = 0;
62   int action_line = 0;
63   int bufferSize = 16384;
64
65   File file;
66   Stack files = new Stack();
67
68   StringBuffer userCode   = new StringBuffer();
69   
70   String classCode;
71   String initCode;   
72   String initThrow;
73   String eofCode;
74   String eofThrow;
75   String lexThrow;
76   String eofVal;
77   String scanErrorException;
78   String cupSymbol = "sym";
79
80   StringBuffer actionText = new StringBuffer();
81   StringBuffer string     = new StringBuffer();
82   
83   boolean charCount;
84   boolean lineCount;
85   boolean columnCount;
86   boolean cupCompatible;  
87   boolean cupDebug;
88   boolean isInteger;
89   boolean isIntWrap;
90   boolean isYYEOF;
91   boolean notUnix;
92   boolean isPublic;
93   boolean isFinal;
94   boolean isAbstract;
95   boolean bolUsed;
96   boolean standalone;
97   boolean debugOption;
98   boolean useRowMap = Options.gen_method == Options.PACK || Options.gen_method == Options.TABLE;
99   boolean packed = Options.gen_method == Options.PACK;
100   boolean caseless;
101   boolean inclusive_states;
102   boolean eofclose;
103     
104   String isImplementing;
105   String isExtending;
106   String className = "Yylex";
107   String functionName;
108   String tokenType;
109   String visibility = "public";
110     
111   Vector /* String */ ctorArgs = new Vector();
112   Vector /* String */ ctorTypes = new Vector();
113     
114   LexicalStates states = new LexicalStates();
115
116   Vector actions = new Vector();
117   
118   private int nextState;
119
120   boolean macroDefinition;
121
122   Timer t = new Timer();
123
124   public int currentLine() {
125     return yyline;
126   }    
127
128   public void setFile(File file) {
129     this.file = file;
130   }
131
132   private Symbol symbol(int type, Object value) {
133     return new Symbol(type, yyline, yycolumn, value);
134   }
135
136   private Symbol symbol(int type) {
137     return new Symbol(type, yyline, yycolumn);
138   }
139    
140   // updates line and column count to the beginning of the first
141   // non whitespace character in yytext, but leaves yyline+yycolumn 
142   // untouched
143   private Symbol symbol_countUpdate(int type, Object value) {
144      int lc = yyline;
145      int cc = yycolumn;
146      String text = yytext();
147
148      for (int i=0; i < text.length(); i++) {
149       char c = text.charAt(i);
150
151       if (c != '\n' && c != '\r' && c != ' ' && c != '\t' ) 
152         return new Symbol(type, lc, cc, value);
153
154       if (c == '\n') {
155         lc++;
156         cc = 0;
157       }
158       else
159         cc++;
160     }
161    
162     return new Symbol(type, yyline, yycolumn, value);
163   }
164
165   private String makeMacroIdent() {
166     String matched = yytext().trim();
167     return matched.substring(1, matched.length()-1).trim();
168   }
169
170   public static String conc(Object a, Object b) {
171     if (a == null && b == null) return null;
172     if (a == null) return b.toString();
173     if (b == null) return a.toString();
174     
175     return a.toString()+b.toString();
176   }
177
178   public static String concExc(Object a, Object b) {
179     if (a == null && b == null) return null;
180     if (a == null) return b.toString();
181     if (b == null) return a.toString();
182     
183     return a.toString()+", "+b.toString();
184   }
185 %}
186
187 %init{
188   states.insert("YYINITIAL", true);
189 %init}
190
191
192 Digit      = [0-9]
193 HexDigit   = [0-9a-fA-F]
194 OctDigit   = [0-7]
195
196 Number     = {Digit}+
197 HexNumber  = \\ x {HexDigit} {2}
198 Unicode    = \\ u {HexDigit} {1, 4}
199 OctNumber  = \\ [0-3]? {OctDigit} {1, 2}  
200
201 // see http://www.unicode.org/unicode/reports/tr18/
202 WSP        = [ \t\b]
203 WSPNL      = [\u2028\u2029\u000A\u000B\u000C\u000D\u0085\t\b\ ]
204 NL         = [\u2028\u2029\u000A\u000B\u000C\u000D\u0085] | \u000D\u000A
205 NNL        = [^\u2028\u2029\u000A\u000B\u000C\u000D\u0085]
206
207 Ident      = {IdentStart} {IdentPart}*
208 QualIdent  = {Ident} ( {WSP}* "." {WSP}* {Ident} )*
209 QUIL       = {QualIdent} ( {WSP}* "," {WSP}* {QualIdent} )*
210 Array      = "[" {WSP}* "]"
211 ParamPart  = {IdentStart}|{IdentPart}|"<"|">"|","|{WSP}|"&"|"?"|"."
212 GenParam   = "<" {ParamPart}+ ">"
213 ClassT     = {Ident} ({WSP}* {GenParam})?
214 QClassT    = {QualIdent} ({WSP}* {GenParam})?
215 ArrType    = ({GenParam} {WSP}*)? {QClassT} ({WSP}* {Array})*
216
217 IdentStart = [:jletter:]
218 IdentPart  = [:jletterdigit:]
219
220 JFlexCommentChar = [^*/]|"/"+[^*/]|"*"+[^*/]
221 JFlexComment = {JFlexCommentChar}*
222
223 /* Java comments */
224 JavaComment = {TraditionalComment}|{EndOfLineComment}
225 TraditionalComment = "/*"{CommentContent}\*+"/"
226 EndOfLineComment = "//".*{NL}
227
228 CommentContent = ([^*]|\*+[^*/])*
229
230 StringCharacter = [^\u2028\u2029\u000A\u000B\u000C\u000D\u0085\"\\]
231
232 CharLiteral = \'([^\u2028\u2029\u000A\u000B\u000C\u000D\u0085\'\\]|{EscapeSequence})\'
233 StringLiteral = \"({StringCharacter}|{EscapeSequence})*\"
234
235 EscapeSequence = \\[^\u2028\u2029\u000A\u000B\u000C\u000D\u0085]|\\+u{HexDigit}{4}|\\[0-3]?{OctDigit}{1,2}
236
237 /* \\(b|t|n|f|r|\"|\'|\\|[0-3]?{OctDigit}{1,2}|u{HexDigit}{4}) */
238
239 JavaRest = [^\{\}\"\'/]|"/"[^*/]      
240 JavaCode = ({JavaRest}|{StringLiteral}|{CharLiteral}|{JavaComment})+
241
242 %%
243
244 <YYINITIAL> {
245   "%%".*{NL}?              { 
246                              t.start(); 
247                              yybegin(MACROS); 
248                              macroDefinition = true; 
249                              return symbol(USERCODE,userCode); 
250                            }
251   .*{NL}                   { userCode.append(yytext()); }            
252   .*                       { return symbol(EOF); }
253 }
254
255 <MACROS>   ("%{"|"%init{"|"%initthrow{"|"%eof{"|"%eofthrow{"|"%yylexthrow{"|"%eofval{").*{NL}
256                                      { string.setLength(0); yybegin(COPY); }
257 <COPY> {
258   "%}".*{NL}                    { classCode = conc(classCode,string);  yybegin(MACROS);  }
259   "%init}".*{NL}                { initCode = conc(initCode,string);    yybegin(MACROS);  }
260   "%initthrow}".*{NL}           { initThrow = concExc(initThrow,string);  yybegin(MACROS); }
261   "%eof}".*{NL}                 { eofCode = conc(eofCode,string); yybegin(MACROS); }
262   "%eofthrow}".*{NL}            { eofThrow = concExc(eofThrow,string); yybegin(MACROS); }
263   "%yylexthrow}".*{NL}          { lexThrow = concExc(lexThrow,string); yybegin(MACROS); }
264   "%eofval}".*{NL}              { eofVal = string.toString(); yybegin(MACROS); }
265
266   .*{NL}                        { string.append(yytext()); }
267
268   <<EOF>>                       { throw new ScannerException(file,ErrorMessages.EOF_IN_MACROS); }
269 }
270
271
272 <MACROS> ^"%s" ("tate" "s"?)? {WSP}+   { inclusive_states = true; yybegin(STATELIST); }
273 <MACROS> ^"%x" ("state" "s"?)? {WSP}+  { inclusive_states = false; yybegin(STATELIST); }
274 <STATELIST> {
275   {Ident}                             { states.insert(yytext(),inclusive_states); }
276   ([\ \t]*","[\ \t]*)|([\ \t]+)       { }
277   {NL}                                { yybegin(MACROS);  }
278   <<EOF>>                       { throw new ScannerException(file,ErrorMessages.EOF_IN_MACROS); }
279 }
280
281 <MACROS> {
282   "%char"                     { charCount = true;  }
283   "%line"                     { lineCount = true;  }
284   "%column"                   { columnCount = true; }
285   "%byaccj"                   { isInteger = true;
286                                 if (eofVal == null)
287                                   eofVal = "return 0;";
288                                 eofclose = true;
289                               }
290   "%cup"                      { cupCompatible = true;  
291                                 isImplementing = concExc(isImplementing, "java_cup.runtime.Scanner");
292                                 if (functionName == null)
293                                   functionName = "next_token";
294                                 if (tokenType == null)
295                                   tokenType = "java_cup.runtime.Symbol";
296                                 if (eofVal == null)
297                                   eofVal = "return new java_cup.runtime.Symbol("+cupSymbol+".EOF);";
298                                 if (!Options.jlex) eofclose = true;
299                               }
300   "%cupsym"{WSP}+{QualIdent} {WSP}*  { cupSymbol = yytext().substring(8).trim(); 
301                                 if (cupCompatible) Out.warning(ErrorMessages.CUPSYM_AFTER_CUP, yyline); }
302   "%cupsym"{WSP}+{NNL}*       { throw new ScannerException(file,ErrorMessages.QUIL_CUPSYM, yyline); }
303   "%cupdebug"                 { cupDebug = true; }
304   "%eofclose"({WSP}+"true")?  { eofclose = true; }
305   "%eofclose"({WSP}+"false")  { eofclose = false; }
306   "%class"{WSP}+{ClassT} {WSP}*     { className = yytext().substring(7).trim();  }
307   "%ctorarg"{WSP}+{ArrType}{WSP}+   { yybegin(CTOR_ARG); ctorTypes.add(yytext().substring(8).trim()); }
308   "%function"{WSP}+{Ident} {WSP}*   { functionName = yytext().substring(10).trim(); }
309   "%type"{WSP}+{ArrType} {WSP}*     { tokenType = yytext().substring(6).trim(); }
310   "%integer"|"%int"           { isInteger = true;  }
311   "%intwrap"                  { isIntWrap = true;  }
312   "%yyeof"                    { isYYEOF = true;  }
313   "%notunix"                  { notUnix = true;  }
314   "%7bit"                     { return symbol(ASCII); }
315   "%full"|"%8bit"             { return symbol(FULL); }
316   "%unicode"|"%16bit"         { return symbol(UNICODE);  }
317   "%caseless"|"%ignorecase"   { caseless = true; }
318   "%implements"{WSP}+.*       { isImplementing = concExc(isImplementing, yytext().substring(12).trim());  }
319   "%extends"{WSP}+{QClassT}{WSP}* { isExtending = yytext().substring(9).trim(); }
320   "%public"                   { isPublic = true; }
321   "%apiprivate"               { visibility = "private"; Skeleton.makePrivate(); }
322   "%final"                    { isFinal = true; }
323   "%abstract"                 { isAbstract = true; }
324   "%debug"                    { debugOption = true; }
325   "%standalone"               { standalone = true; isInteger = true; }
326   "%switch"                   { packed = false; useRowMap = false; }
327   "%table"                    { packed = false; useRowMap = true; }
328   "%pack"                     { packed = true; useRowMap = true; }
329   "%include" {WSP}+ .*        { File f = new File(yytext().substring(9).trim());
330                                 if ( !f.canRead() )
331                                   throw new ScannerException(file,ErrorMessages.NOT_READABLE, yyline); 
332                                 // check for cycle
333                                 if (files.search(f) > 0)
334                                   throw new ScannerException(file,ErrorMessages.FILE_CYCLE, yyline);
335                                 try {
336                                   yypushStream( new FileReader(f) );
337                                   files.push(file);
338                                   file = f;
339                                   Out.println("Including \""+file+"\"");
340                                 }
341                                 catch (FileNotFoundException e) {
342                                   throw new ScannerException(file,ErrorMessages.NOT_READABLE, yyline); 
343                                 } 
344                               }
345   "%buffer" {WSP}+ {Number} {WSP}*   { bufferSize = Integer.parseInt(yytext().substring(8).trim()); }
346   "%buffer" {WSP}+ {NNL}*     { throw new ScannerException(file,ErrorMessages.NO_BUFFER_SIZE, yyline); }
347   "%initthrow" {WSP}+ {QUIL} {WSP}* { initThrow = concExc(initThrow,yytext().substring(11).trim()); }
348   "%initthrow" {WSP}+ {NNL}*  { throw new ScannerException(file,ErrorMessages.QUIL_INITTHROW, yyline); }
349   "%eofthrow"  {WSP}+ {QUIL} {WSP}*  { eofThrow = concExc(eofThrow,yytext().substring(10).trim()); }
350   "%eofthrow"  {WSP}+ {NNL}*  { throw new ScannerException(file,ErrorMessages.QUIL_EOFTHROW, yyline); }
351   "%yylexthrow"{WSP}+ {QUIL} {WSP}*  { lexThrow = concExc(lexThrow,yytext().substring(12).trim()); }
352   "%throws"    {WSP}+ {QUIL} {WSP}*  { lexThrow = concExc(lexThrow,yytext().substring(8).trim()); }
353   "%yylexthrow"{WSP}+ {NNL}*  { throw new ScannerException(file,ErrorMessages.QUIL_YYLEXTHROW, yyline); }
354   "%throws"    {WSP}+ {NNL}*  { throw new ScannerException(file,ErrorMessages.QUIL_THROW, yyline); }
355   "%scanerror" {WSP}+ {QualIdent} {WSP}* { scanErrorException = yytext().substring(11).trim(); }
356   "%scanerror" {WSP}+ {NNL}*  { throw new ScannerException(file,ErrorMessages.QUIL_SCANERROR, yyline); }
357
358   {Ident}                     { return symbol(IDENT, yytext()); }
359   "="{WSP}*                   { yybegin(REGEXP); return symbol(EQUALS); }
360
361   "/*"                        { nextState = MACROS; yybegin(COMMENT); }
362   
363   {EndOfLineComment}          { }
364
365   /* no {NL} at the end of this expression, because <REGEXPSTART> 
366      needs at least one {WSPNL} to start a regular expression! */   
367   ^"%%" {NNL}*                { macroDefinition = false; yybegin(REGEXPSTART); return symbol(DELIMITER); }
368   "%"{Ident}                  { throw new ScannerException(file,ErrorMessages.UNKNOWN_OPTION, yyline, yycolumn); }
369   "%"                         { throw new ScannerException(file,ErrorMessages.UNKNOWN_OPTION, yyline, yycolumn); }
370   ^{WSP}+"%"                  { Out.warning(ErrorMessages.NOT_AT_BOL, yyline); yypushback(1); }
371
372   {WSP}+                      { }
373   {NL}+                       { }                        
374   <<EOF>>                     { if ( yymoreStreams() ) {
375                                   file = (File) files.pop();
376                                   yypopStream();
377                                 }
378                                 else
379                                   throw new ScannerException(file,ErrorMessages.EOF_IN_MACROS); 
380                               }
381 }
382
383 <CTOR_ARG> {
384   {Ident} {WSP}*   { yybegin(MACROS); ctorArgs.add(yytext().trim()); }
385   [^]              { throw new ScannerException(file,ErrorMessages.CTOR_ARG,yyline,yycolumn); }
386 }
387
388 <REGEXPSTART> {
389   {WSPNL}* "/*"               { nextState = REGEXPSTART; yybegin(COMMENT); }
390   {WSPNL}+                    { yybegin(REGEXP); }
391   {WSPNL}* "<"                { yybegin(STATES); return symbol_countUpdate(LESSTHAN, null); }
392   {WSPNL}* "}"                { return symbol_countUpdate(RBRACE, null); }
393   {WSPNL}* "//" {NNL}*        { }  
394   {WSPNL}* "<<EOF>>" {WSPNL}* "{" 
395                               { actionText.setLength(0); yybegin(JAVA_CODE); 
396                                 Symbol s = symbol_countUpdate(EOFRULE, null);
397                                 action_line = s.left+1; 
398                                 return s;
399                               }
400 }
401
402 <STATES> {
403   {Ident}                     { return symbol(IDENT, yytext()); }
404   ","                         { return symbol(COMMA); }
405   {WSPNL}+                    { }
406
407   // "{" will be caught in REGEXP  
408   ">"{WSPNL}*                 { yybegin(REGEXP); return symbol(MORETHAN); }
409
410   <<EOF>>                     { throw new ScannerException(file,ErrorMessages.EOF_IN_STATES); }
411 }
412
413
414 <REGEXP> {
415   "<<EOF>>" {WSPNL}+ "{"  { actionText.setLength(0); yybegin(JAVA_CODE); action_line = yyline+1; return symbol(EOFRULE); }
416   "<<EOF>>"               { throw new ScannerException(file,ErrorMessages.EOF_WO_ACTION); }
417
418   {WSPNL}*"|"{WSP}*$      { if (macroDefinition) {
419                               yybegin(EATWSPNL);
420                               return symbol(BAR); 
421                             }
422                             else { 
423                               yybegin(REGEXPSTART); 
424                               return symbol(NOACTION); 
425                             }
426                           }
427
428   // stategroup
429   "{"          { yybegin(REGEXPSTART); return symbol(LBRACE); }
430
431   {WSPNL}*"|"  { return symbol(BAR); }
432
433   {WSPNL}*\"   { string.setLength(0); nextState = REGEXP; yybegin(STRING_CONTENT); }
434   {WSPNL}*"!"  { return symbol(BANG); }
435   {WSPNL}*"~"  { return symbol(TILDE); }
436   {WSPNL}*"("  { return symbol(OPENBRACKET); }
437   {WSPNL}*")"  { return symbol(CLOSEBRACKET); }
438   {WSPNL}*"*"  { return symbol(STAR); }
439   {WSPNL}*"+"  { return symbol(PLUS); }
440   {WSPNL}*"?"  { return symbol(QUESTION); }
441   {WSPNL}*"$"  { return symbol(DOLLAR); }
442   {WSPNL}*"^"  { bolUsed = true; return symbol(HAT); }
443   {WSPNL}*"."  { return symbol(POINT); }
444   {WSPNL}*"["  { yybegin(CHARCLASS); return symbol(OPENCLASS); }
445   {WSPNL}*"/"  { return symbol(LOOKAHEAD); }
446   
447   {WSPNL}* "{" {WSP}* {Ident} {WSP}* "}" { return symbol_countUpdate(MACROUSE, makeMacroIdent()); }
448   {WSPNL}* "{" {WSP}* {Number}   { yybegin(REPEATEXP); return symbol(REPEAT, new Integer(yytext().trim().substring(1).trim())); }
449
450   {WSPNL}+ "{"    { actionText.setLength(0); yybegin(JAVA_CODE); action_line = yyline+1; return symbol(REGEXPEND); }
451   {NL}            { if (macroDefinition) { yybegin(MACROS); } return symbol(REGEXPEND); }
452
453   {WSPNL}*"/*"    { nextState = REGEXP; yybegin(COMMENT); }
454
455   {WSPNL}*"//"{NNL}*  { }
456
457   {WSP}+          { }
458
459   <CHARCLASS> {
460     {WSPNL}*"[:jletter:]"  { return symbol(JLETTERCLASS); }
461     {WSPNL}*"[:jletterdigit:]" { return symbol(JLETTERDIGITCLASS); }
462     {WSPNL}*"[:letter:]"     { return symbol(LETTERCLASS); }
463     {WSPNL}*"[:digit:]"      { return symbol(DIGITCLASS); }
464     {WSPNL}*"[:uppercase:]"  { return symbol(UPPERCLASS); }
465     {WSPNL}*"[:lowercase:]"  { return symbol(LOWERCLASS); }
466     {WSPNL}*"[:unicode_unassigned:]"  { return symbol(UNICODE_UNASSIGNED); }
467     {WSPNL}*"[:unicode_uppercase_letter:]"  { return symbol(UNICODE_UPPERCASE_LETTER); }
468     {WSPNL}*"[:unicode_lowercase_letter:]"  { return symbol(UNICODE_LOWERCASE_LETTER); }
469     {WSPNL}*"[:unicode_titlecase_letter:]"  { return symbol(UNICODE_TITLECASE_LETTER); }
470     {WSPNL}*"[:unicode_modifier_letter:]"  { return symbol(UNICODE_MODIFIER_LETTER); }
471     {WSPNL}*"[:unicode_other_letter:]"  { return symbol(UNICODE_OTHER_LETTER); }
472     {WSPNL}*"[:unicode_non_spacing_mark:]"  { return symbol(UNICODE_NON_SPACING_MARK); }
473     {WSPNL}*"[:unicode_enclosing_mark:]"  { return symbol(UNICODE_ENCLOSING_MARK); }
474     {WSPNL}*"[:unicode_combining_spacing_mark:]"  { return symbol(UNICODE_COMBINING_SPACING_MARK); }
475     {WSPNL}*"[:unicode_decimal_digit_number:]"  { return symbol(UNICODE_DECIMAL_DIGIT_NUMBER); }
476     {WSPNL}*"[:unicode_letter_number:]"  { return symbol(UNICODE_LETTER_NUMBER); }
477     {WSPNL}*"[:unicode_other_number:]"  { return symbol(UNICODE_OTHER_NUMBER); }
478     {WSPNL}*"[:unicode_space_separator:]"  { return symbol(UNICODE_SPACE_SEPARATOR); }
479     {WSPNL}*"[:unicode_line_separator:]"  { return symbol(UNICODE_LINE_SEPARATOR); }
480     {WSPNL}*"[:unicode_paragraph_separator:]"  { return symbol(UNICODE_PARAGRAPH_SEPARATOR); }
481     {WSPNL}*"[:unicode_control:]"  { return symbol(UNICODE_CONTROL); }
482     {WSPNL}*"[:unicode_format:]"  { return symbol(UNICODE_FORMAT); }
483     {WSPNL}*"[:unicode_private_use:]"  { return symbol(UNICODE_PRIVATE_USE); }
484     {WSPNL}*"[:unicode_surrogate:]"  { return symbol(UNICODE_SURROGATE); }
485     {WSPNL}*"[:unicode_dash_punctuation:]"  { return symbol(UNICODE_DASH_PUNCTUATION); }
486     {WSPNL}*"[:unicode_start_punctuation:]"  { return symbol(UNICODE_START_PUNCTUATION); }
487     {WSPNL}*"[:unicode_end_punctuation:]"  { return symbol(UNICODE_END_PUNCTUATION); }
488     {WSPNL}*"[:unicode_connector_punctuation:]"  { return symbol(UNICODE_CONNECTOR_PUNCTUATION); }
489     {WSPNL}*"[:unicode_other_punctuation:]"  { return symbol(UNICODE_OTHER_PUNCTUATION); }
490     {WSPNL}*"[:unicode_math_symbol:]"  { return symbol(UNICODE_MATH_SYMBOL); }
491     {WSPNL}*"[:unicode_currency_symbol:]"  { return symbol(UNICODE_CURRENCY_SYMBOL); }
492     {WSPNL}*"[:unicode_modifier_symbol:]"  { return symbol(UNICODE_MODIFIER_SYMBOL); }
493     {WSPNL}*"[:unicode_other_symbol:]"  { return symbol(UNICODE_OTHER_SYMBOL); }
494     {WSPNL}*"[:unicode_initial_quote_punctuation:]"  { return symbol(UNICODE_INITIAL_QUOTE_PUNCTUATION); }
495     {WSPNL}*"[:unicode_final_quote_punctuation:]"  { return symbol(UNICODE_FINAL_QUOTE_PUNCTUATION); }
496   }
497
498   . { return symbol(CHAR, new Character(yytext().charAt(0))); }
499 }
500
501 <EATWSPNL> {WSPNL}+  { yybegin(REGEXP); }
502
503
504 <REPEATEXP> {
505   "}"          { yybegin(REGEXP); return symbol(RBRACE); }
506   "," {WSP}* {Number}  { return symbol(REPEAT, new Integer(yytext().substring(1).trim())); }
507   {WSP}+       { }
508
509   <<EOF>>                 { throw new ScannerException(file,ErrorMessages.EOF_IN_REGEXP); }
510 }
511
512 <CHARCLASS> {
513   "{"{Ident}"}" { return symbol(MACROUSE, yytext().substring(1,yytext().length()-1)); }
514   "["  { balance++; return symbol(OPENCLASS); }
515   "]"  { if (balance > 0) balance--; else yybegin(REGEXP); return symbol(CLOSECLASS); }
516   "^"  { return symbol(HAT); }
517   "-"  { return symbol(DASH); }
518
519   // this is a hack to keep JLex compatibilty with char class 
520   // expressions like [+-]
521   "-]" { yypushback(1); yycolumn--; return symbol(CHAR, new Character(yytext().charAt(0))); }  
522
523   \"   { string.setLength(0); nextState = CHARCLASS; yybegin(STRING_CONTENT); }
524
525   .    { return symbol(CHAR, new Character(yytext().charAt(0))); }
526
527   \n   { throw new ScannerException(file,ErrorMessages.EOL_IN_CHARCLASS,yyline,yycolumn); }
528
529   <<EOF>>     { throw new ScannerException(file,ErrorMessages.EOF_IN_REGEXP); }
530 }
531
532 <STRING_CONTENT> {
533   \"       { yybegin(nextState); return symbol(STRING, string.toString()); }
534   \\\"     { string.append('\"'); }
535   [^\"\\\u2028\u2029\u000A\u000B\u000C\u000D\u0085]+ { string.append(yytext()); }
536
537   {NL}     { throw new ScannerException(file,ErrorMessages.UNTERMINATED_STR, yyline, yycolumn); }
538
539   {HexNumber} { string.append( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16)); }
540   {Unicode}   { string.append( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16)); }
541   {OctNumber} { string.append( (char) Integer.parseInt(yytext().substring(1,yytext().length()), 8)); }
542
543   \\b { string.append('\b'); }
544   \\n { string.append('\n'); }
545   \\t { string.append('\t'); }
546   \\f { string.append('\f'); }
547   \\r { string.append('\r'); }
548
549   \\. { string.append(yytext().charAt(1)); }
550
551   <<EOF>>     { throw new ScannerException(file,ErrorMessages.EOF_IN_STRING); }
552 }
553
554
555 <REGEXP, CHARCLASS> {
556   {HexNumber} { return symbol(CHAR, new Character( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16))); }
557   {Unicode} { return symbol(CHAR, new Character( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16))); }
558   {OctNumber} { return symbol(CHAR, new Character( (char) Integer.parseInt(yytext().substring(1,yytext().length()), 8))); }
559
560   \\b { return symbol(CHAR,new Character('\b')); }
561   \\n { return symbol(CHAR,new Character('\n')); }
562   \\t { return symbol(CHAR,new Character('\t')); }
563   \\f { return symbol(CHAR,new Character('\f')); }
564   \\r { return symbol(CHAR,new Character('\r')); }
565
566   \\. { return symbol(CHAR, new Character(yytext().charAt(1))); }
567 }
568
569
570 <JAVA_CODE> {
571   "{"        { balance++; actionText.append('{'); }
572   "}"        { if (balance > 0) {
573                  balance--;     
574                  actionText.append('}'); 
575                }
576                else {
577                  yybegin(REGEXPSTART); 
578                  Action a = new Action(actionText.toString(), action_line);
579                  actions.addElement(a);
580                  return symbol(ACTION, a);
581                }
582              } 
583            
584   {JavaCode}     { actionText.append(yytext()); } 
585
586   <<EOF>>     { throw new ScannerException(file,ErrorMessages.EOF_IN_ACTION, action_line-1); }
587 }
588
589 <COMMENT> {
590    
591   "/"+ "*"  { commentbalance++; }
592   "*"+ "/"  { if (commentbalance > 0) 
593                 commentbalance--; 
594               else
595                 yybegin(nextState); 
596             }
597   
598   {JFlexComment} { /* ignore */ }
599
600   <<EOF>>     { throw new ScannerException(file,ErrorMessages.EOF_IN_COMMENT); }
601 }
602
603
604 .  { throw new ScannerException(file,ErrorMessages.UNEXPECTED_CHAR, yyline, yycolumn); }
605 \n { throw new ScannerException(file,ErrorMessages.UNEXPECTED_NL, yyline, yycolumn); }
606
607 <<EOF>>  { if ( yymoreStreams() ) {
608              file = (File) files.pop();
609              yypopStream();
610            }
611            else 
612              return symbol(EOF); }