move JFlex to CE source

diff --git a/tools/lexer/idea-flex.skeleton b/tools/lexer/idea-flex.skeleton
new file mode 100644 (file)
index 0000000..a91ec0c
--- /dev/null
+++ b/tools/lexer/idea-flex.skeleton
@@ -0,0 +1,263 @@
+  /** initial size of the lookahead buffer */
+--- private static final int ZZ_BUFFERSIZE = ...;
+
+  /** lexical states */
+---  lexical states, charmap
+
+  /* error codes */
+  private static final int ZZ_UNKNOWN_ERROR = 0;
+  private static final int ZZ_NO_MATCH = 1;
+  private static final int ZZ_PUSHBACK_2BIG = 2;
+  private static final char[] EMPTY_BUFFER = new char[0];
+  private static final int YYEOF = -1;
+  private static java.io.Reader zzReader = null; // Fake
+
+  /* error messages for the codes above */
+  private static final String ZZ_ERROR_MSG[] = {
+    "Unkown internal scanner error",
+    "Error: could not match input",
+    "Error: pushback value was too large"
+  };
+
+--- isFinal list
+  /** the current state of the DFA */
+  private int zzState;
+
+  /** the current lexical state */
+  private int zzLexicalState = YYINITIAL;
+
+  /** this buffer contains the current text to be matched and is
+      the source of the yytext() string */
+  private CharSequence zzBuffer = "";
+
+  /** this buffer may contains the current text array to be matched when it is cheap to acquire it */
+  private char[] zzBufferArray;
+
+  /** the textposition at the last accepting state */
+  private int zzMarkedPos;
+
+  /** the textposition at the last state to be included in yytext */
+  private int zzPushbackPos;
+
+  /** the current text position in the buffer */
+  private int zzCurrentPos;
+
+  /** startRead marks the beginning of the yytext() string in the buffer */
+  private int zzStartRead;
+
+  /** endRead marks the last character in the buffer, that has been read
+      from input */
+  private int zzEndRead;
+
+  /**
+   * zzAtBOL == true <=> the scanner is currently at the beginning of a line
+   */
+  private boolean zzAtBOL = true;
+
+  /** zzAtEOF == true <=> the scanner is at the EOF */
+  private boolean zzAtEOF;
+
+--- user class code
+
+--- constructor declaration
+
+  public final int getTokenStart(){
+    return zzStartRead;
+  }
+
+  public final int getTokenEnd(){
+    return getTokenStart() + yylength();
+  }
+
+  public void reset(CharSequence buffer, int start, int end,int initialState){
+    zzBuffer = buffer;
+    zzBufferArray = com.intellij.util.text.CharArrayUtil.fromSequenceWithoutCopying(buffer);
+    zzCurrentPos = zzMarkedPos = zzStartRead = start;
+    zzPushbackPos = 0;
+    zzAtEOF  = false;
+    zzAtBOL = true;
+    zzEndRead = end;
+    yybegin(initialState);
+  }
+
+  // For Demetra compatibility
+  public void reset(CharSequence buffer, int initialState){
+    zzBuffer = buffer;
+    zzBufferArray = null; 
+    zzCurrentPos = zzMarkedPos = zzStartRead = 0;
+    zzPushbackPos = 0;
+    zzAtEOF = false;
+    zzAtBOL = true;
+    zzEndRead = buffer.length();
+    yybegin(initialState);
+  }
+
+  /**
+   * Refills the input buffer.
+   *
+   * @return      <code>false</code>, iff there was new input.
+   *
+   * @exception   java.io.IOException  if any I/O-Error occurs
+   */
+  private boolean zzRefill() throws java.io.IOException {
+    return true;
+  }
+
+
+  /**
+   * Returns the current lexical state.
+   */
+  public final int yystate() {
+    return zzLexicalState;
+  }
+
+
+  /**
+   * Enters a new lexical state
+   *
+   * @param newState the new lexical state
+   */
+  public final void yybegin(int newState) {
+    zzLexicalState = newState;
+  }
+
+
+  /**
+   * Returns the text matched by the current regular expression.
+   */
+  public final CharSequence yytext() {
+    return zzBuffer.subSequence(zzStartRead, zzMarkedPos);
+  }
+
+
+  /**
+   * Returns the character at position <tt>pos</tt> from the
+   * matched text.
+   *
+   * It is equivalent to yytext().charAt(pos), but faster
+   *
+   * @param pos the position of the character to fetch.
+   *            A value from 0 to yylength()-1.
+   *
+   * @return the character at position pos
+   */
+  public final char yycharat(int pos) {
+    return zzBufferArray != null ? zzBufferArray[zzStartRead+pos]:zzBuffer.charAt(zzStartRead+pos);
+  }
+
+
+  /**
+   * Returns the length of the matched text region.
+   */
+  public final int yylength() {
+    return zzMarkedPos-zzStartRead;
+  }
+
+
+  /**
+   * Reports an error that occured while scanning.
+   *
+   * In a wellformed scanner (no or only correct usage of
+   * yypushback(int) and a match-all fallback rule) this method
+   * will only be called with things that "Can't Possibly Happen".
+   * If this method is called, something is seriously wrong
+   * (e.g. a JFlex bug producing a faulty scanner etc.).
+   *
+   * Usual syntax/scanner level error handling should be done
+   * in error fallback rules.
+   *
+   * @param   errorCode  the code of the errormessage to display
+   */
+--- zzScanError declaration
+    String message;
+    try {
+      message = ZZ_ERROR_MSG[errorCode];
+    }
+    catch (ArrayIndexOutOfBoundsException e) {
+      message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
+    }
+
+--- throws clause
+  }
+
+
+  /**
+   * Pushes the specified amount of characters back into the input stream.
+   *
+   * They will be read again by then next call of the scanning method
+   *
+   * @param number  the number of characters to be read again.
+   *                This number must not be greater than yylength()!
+   */
+--- yypushback decl (contains zzScanError exception)
+    if ( number > yylength() )
+      zzScanError(ZZ_PUSHBACK_2BIG);
+
+    zzMarkedPos -= number;
+  }
+
+
+--- zzDoEOF
+  /**
+   * Resumes scanning until the next regular expression is matched,
+   * the end of input is encountered or an I/O-Error occurs.
+   *
+   * @return      the next token
+   * @exception   java.io.IOException  if any I/O-Error occurs
+   */
+--- yylex declaration
+    int zzInput;
+    int zzAction;
+
+    // cached fields:
+    int zzCurrentPosL;
+    int zzMarkedPosL;
+    int zzEndReadL = zzEndRead;
+    CharSequence zzBufferL = zzBuffer;
+    char[] zzBufferArrayL = zzBufferArray;
+    char [] zzCMapL = ZZ_CMAP;
+
+--- local declarations
+
+    while (true) {
+      zzMarkedPosL = zzMarkedPos;
+
+--- start admin (line, char, col count)
+      zzAction = -1;
+
+      zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
+
+--- start admin (lexstate etc)
+
+      zzForAction: {
+        while (true) {
+
+--- next input, line, col, char count, next transition, isFinal action
+            zzAction = zzState;
+            zzMarkedPosL = zzCurrentPosL;
+--- line count update
+          }
+
+        }
+      }
+
+      // store back cached position
+      zzMarkedPos = zzMarkedPosL;
+--- char count update
+
+--- actions
+        default:
+          if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
+            zzAtEOF = true;
+--- eofvalue
+          }
+          else {
+--- no match
+          }
+      }
+    }
+  }
+
+--- main
+
+}

diff --git a/tools/lexer/jflex-1.4/.classpath b/tools/lexer/jflex-1.4/.classpath
new file mode 100644 (file)
index 0000000..c7ff90e
--- /dev/null
+++ b/tools/lexer/jflex-1.4/.classpath
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+    <classpathentry kind="src" path="src"/>
+    <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+    <classpathentry kind="var" path="ECLIPSE_HOME/plugins/org.apache.ant_1.5.2/ant.jar"/>
+    <classpathentry kind="var"
+        path="ECLIPSE_HOME/plugins/org.junit_3.8.1/junit.jar" sourcepath="ECLIPSE_HOME/plugins/org.junit_3.8.1/src.jar"/>
+    <classpathentry kind="output" path="classes"/>
+</classpath>

diff --git a/tools/lexer/jflex-1.4/.project b/tools/lexer/jflex-1.4/.project
new file mode 100644 (file)
index 0000000..3160ec9
--- /dev/null
+++ b/tools/lexer/jflex-1.4/.project
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+       <name>jflex</name>
+       <comment></comment>
+       <projects>
+       </projects>
+       <buildSpec>
+               <buildCommand>
+                       <name>org.eclipse.jdt.core.javabuilder</name>
+                       <arguments>
+                       </arguments>
+               </buildCommand>
+       </buildSpec>
+       <natures>
+               <nature>org.eclipse.jdt.core.javanature</nature>
+       </natures>
+</projectDescription>

diff --git a/tools/lexer/jflex-1.4/COPYRIGHT b/tools/lexer/jflex-1.4/COPYRIGHT
new file mode 100644 (file)
index 0000000..0109c54
--- /dev/null
+++ b/tools/lexer/jflex-1.4/COPYRIGHT
@@ -0,0 +1,303 @@
+JFlex - Copying, Warranty & License
+
+  
+
+JFlex is free software, published under the terms of the GNU General Public License. 
+
+There is absolutely NO WARRANTY for JFlex, its code and its documentation. 
+  
+The code generated by JFlex inherits the copyright of the specification it
+was produced from. If it was your specification, you may use the generated 
+code without restriction.
+
+  
+
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+
+
+0.
+ This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+
+
+1.
+ You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+
+2.
+ You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+
+
+     a)
+     You must cause the modified files to carry prominent notices
+     stating that you changed the files and the date of any change.
+
+
+     b)
+     You must cause any work that you distribute or publish, that in
+     whole or in part contains or is derived from the Program or any
+     part thereof, to be licensed as a whole at no charge to all third
+     parties under the terms of this License.
+
+
+     c)
+     If the modified program normally reads commands interactively
+     when run, you must cause it, when started running for such
+     interactive use in the most ordinary way, to print or display an
+     announcement including an appropriate copyright notice and a
+     notice that there is no warranty (or else, saying that you provide
+     a warranty) and that users may redistribute the program under
+     these conditions, and telling the user how to view a copy of this
+     License.  (Exception: if the Program itself is interactive but
+     does not normally print such an announcement, your work based on
+     the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+
+
+3.
+ You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+
+
+     a)
+     Accompany it with the complete corresponding machine-readable
+     source code, which must be distributed under the terms of Sections
+     1 and 2 above on a medium customarily used for software interchange; or,
+
+
+     b)
+     Accompany it with a written offer, valid for at least three
+     years, to give any third party, for a charge no more than your
+     cost of physically performing source distribution, a complete
+     machine-readable copy of the corresponding source code, to be
+     distributed under the terms of Sections 1 and 2 above on a medium
+     customarily used for software interchange; or,
+
+
+     c)
+     Accompany it with the information you received as to the offer
+     to distribute corresponding source code.  (This alternative is
+     allowed only for noncommercial distribution and only if you
+     received the program in object code or executable form with such
+     an offer, in accord with Subsection b above.)
+
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+4.
+ You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+
+
+5.
+ You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+
+
+6.
+ Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+
+
+7.
+ If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+
+8.
+ If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+
+
+9.
+ The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+
+
+
+10.
+ If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+
+
+NO WARRANTY
+
+
+
+11.
+ BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+
+
+12.
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+
+
+
+END OF TERMS AND CONDITIONS

diff --git a/tools/lexer/jflex-1.4/JFlex-Changes.txt b/tools/lexer/jflex-1.4/JFlex-Changes.txt
new file mode 100644 (file)
index 0000000..9048b81
--- /dev/null
+++ b/tools/lexer/jflex-1.4/JFlex-Changes.txt
@@ -0,0 +1,12 @@
+The following changes to JFlex 1.4.1 have been done by JetBrains developers:
+
+July 26, 2005:
+
+ * JFlexTask.java: Added setCharAt() method for setting the -charat option from Ant
+   builds.
+
+July 8, 2005:
+
+ * Main.java, Options.java, Emitter.java: Added new command line option -charat.
+   The option replaces array index access when accessing the zzBufferL array with 
+   calls to the .charAt() method.

diff --git a/tools/lexer/jflex-1.4/bin/jflex b/tools/lexer/jflex-1.4/bin/jflex
new file mode 100644 (file)
index 0000000..80f7b76
--- /dev/null
+++ b/tools/lexer/jflex-1.4/bin/jflex
@@ -0,0 +1,51 @@
+#! /bin/bash 
+#
+#     JFlex start script $Revision: 2.0 $
+#
+# if Java is not in your binary path, you need to supply its
+# location in this script. The script automatically finds 
+# JFLEX_HOME when called directly, via binary path, or symbolic
+# link. 
+#
+# Site wide installation: simply make a symlink from e.g.
+# /usr/bin/jflex to this script at its original position
+#
+#===================================================================
+#
+# configurables:
+
+# path to the java interpreter
+JAVA=java
+
+# end configurables
+#
+#===================================================================
+#
+
+# calculate true location
+
+PRG=`type $0`
+PRG=${PRG##* }
+
+# If PRG is a symlink, trace it to the real home directory
+
+while [ -L "$PRG" ]
+do
+    newprg=$(ls -l ${PRG})
+    newprg=${newprg##*-> }
+    [ ${newprg} = ${newprg#/} ] && newprg=${PRG%/*}/${newprg}
+    PRG="$newprg"
+done
+
+PRG=${PRG%/*}
+JFLEX_HOME=${PRG}/.. 
+
+# --------------------------------------------------------------------
+
+export CLASSPATH
+CLASSPATH=$JFLEX_HOME/lib/JFlex.jar
+
+$JAVA JFlex.Main $@
+
+#for more memory:
+#$JAVA -Xmx128m JFlex.Main $@

diff --git a/tools/lexer/jflex-1.4/bin/jflex.bat b/tools/lexer/jflex-1.4/bin/jflex.bat
new file mode 100644 (file)
index 0000000..6ac3ccd
--- /dev/null
+++ b/tools/lexer/jflex-1.4/bin/jflex.bat
@@ -0,0 +1,19 @@
+@echo off
+REM Please adjust the paths JFLEX_HOME and JAVA_HOME to suit your needs
+REM (please do not add a trailing backslash)
+
+set JFLEX_HOME=..
+
+REM only needed for JDK 1.1.x:
+set JAVA_HOME=C:\JAVA
+
+
+REM ------------------------------------------------------------------- 
+
+set CLPATH=%JAVA_HOME%\lib\classes.zip;%JFLEX_HOME%\lib\JFlex.jar
+
+REM for JDK 1.1.x
+REM %JAVA_HOME%\bin\java -classpath %CLPATH% JFlex.Main %1 %2 %3 %4 %5 %6 %7 %8 %9
+
+REM for JDK 1.2
+java -Xmx128m -jar %JFLEX_HOME%\lib\JFlex.jar %1 %2 %3 %4 %5 %6 %7 %8 %9

diff --git a/tools/lexer/jflex-1.4/doc/COPYRIGHT b/tools/lexer/jflex-1.4/doc/COPYRIGHT
new file mode 100644 (file)
index 0000000..0109c54
--- /dev/null
+++ b/tools/lexer/jflex-1.4/doc/COPYRIGHT
@@ -0,0 +1,303 @@
+JFlex - Copying, Warranty & License
+
+  
+
+JFlex is free software, published under the terms of the GNU General Public License. 
+
+There is absolutely NO WARRANTY for JFlex, its code and its documentation. 
+  
+The code generated by JFlex inherits the copyright of the specification it
+was produced from. If it was your specification, you may use the generated 
+code without restriction.
+
+  
+
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+
+
+0.
+ This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+
+
+1.
+ You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+
+2.
+ You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+
+
+     a)
+     You must cause the modified files to carry prominent notices
+     stating that you changed the files and the date of any change.
+
+
+     b)
+     You must cause any work that you distribute or publish, that in
+     whole or in part contains or is derived from the Program or any
+     part thereof, to be licensed as a whole at no charge to all third
+     parties under the terms of this License.
+
+
+     c)
+     If the modified program normally reads commands interactively
+     when run, you must cause it, when started running for such
+     interactive use in the most ordinary way, to print or display an
+     announcement including an appropriate copyright notice and a
+     notice that there is no warranty (or else, saying that you provide
+     a warranty) and that users may redistribute the program under
+     these conditions, and telling the user how to view a copy of this
+     License.  (Exception: if the Program itself is interactive but
+     does not normally print such an announcement, your work based on
+     the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+
+
+3.
+ You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+
+
+     a)
+     Accompany it with the complete corresponding machine-readable
+     source code, which must be distributed under the terms of Sections
+     1 and 2 above on a medium customarily used for software interchange; or,
+
+
+     b)
+     Accompany it with a written offer, valid for at least three
+     years, to give any third party, for a charge no more than your
+     cost of physically performing source distribution, a complete
+     machine-readable copy of the corresponding source code, to be
+     distributed under the terms of Sections 1 and 2 above on a medium
+     customarily used for software interchange; or,
+
+
+     c)
+     Accompany it with the information you received as to the offer
+     to distribute corresponding source code.  (This alternative is
+     allowed only for noncommercial distribution and only if you
+     received the program in object code or executable form with such
+     an offer, in accord with Subsection b above.)
+
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+4.
+ You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+
+
+5.
+ You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+
+
+6.
+ Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+
+
+7.
+ If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+
+8.
+ If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+
+
+9.
+ The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+
+
+
+10.
+ If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+
+
+NO WARRANTY
+
+
+
+11.
+ BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+
+
+12.
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+
+
+
+END OF TERMS AND CONDITIONS

diff --git a/tools/lexer/jflex-1.4/doc/crossref.png b/tools/lexer/jflex-1.4/doc/crossref.png
new file mode 100644 (file)
index 0000000..7dd2ddd
Binary files /dev/null and b/tools/lexer/jflex-1.4/doc/crossref.png differ

diff --git a/tools/lexer/jflex-1.4/doc/footnote.png b/tools/lexer/jflex-1.4/doc/footnote.png
new file mode 100644 (file)
index 0000000..4591c0e
Binary files /dev/null and b/tools/lexer/jflex-1.4/doc/footnote.png differ

diff --git a/tools/lexer/jflex-1.4/doc/jflex_anttask.html b/tools/lexer/jflex-1.4/doc/jflex_anttask.html
new file mode 100644 (file)
index 0000000..942acf2
--- /dev/null
+++ b/tools/lexer/jflex-1.4/doc/jflex_anttask.html
@@ -0,0 +1,165 @@
+<html>
+
+<head>
+<meta http-equiv="Content-Language" content="en-us">
+<title>JFlex Ant Task</title>
+</head>
+
+<body>
+
+<h2><a name="JFlex">JFlex Ant Task</a></h2>
+<p>
+ JFlex can be easily integrated with <a HREF="http://jakarta.apache.org/ant/index.html" 
+ target="_top">Ant</a> build tool. 
+ To use JFlex with Ant, simply copy JFlex.jar to <i>$ANT_HOME/lib/</i> directory.
+</p>
+<h3>Description</h3>
+<p>
+  The JFlex Ant Task invokes the <a HREF="http://jflex.de/" target="_top">JFlex</a> lexical 
+  analyzer generator on a grammar file.
+</p>
+<p>
+ To use the JFlex task, the following line must be placed in the Ant build file:
+</p>
+<pre>&lt;taskdef classname=&quot;JFlex.anttask.JFlexTask&quot; name=&quot;jflex&quot; /&gt;</pre>
+
+<p>
+  The JFlex task requires the <i>file</i> attribute to be set to the source grammar file (*.flex).
+  Unless the target directory is specified with the <i>destdir</i> option, 
+  the generated class will be saved to the same directory where grammar file resides. 
+  Like javac, the JFlex task creates subdirectories in <i>destdir</i> according to the 
+  generated class package.
+</p>
+<p>
+  This task only invokes JFlex if the grammar file is newer than the generated
+  files.
+</p>
+
+<h3>Parameters</h3>
+<table border="1" cellpadding="2" cellspacing="0" width="80%">
+  <tr>
+    <td valign="top"><b>Attribute</b></td>
+    <td valign="top"><b>Description</b></td>
+    <td align="center" valign="top"><b>Required</b></td>
+    <td align="center" valign="top"><b>Default</b></td>
+  </tr>
+  <tr>
+    <td valign="top">file="file"</td>
+    <td valign="top">The grammar file to process.</td>
+    <td valign="top" align="center">Yes</td>
+    <td></td>
+  </tr>
+  <tr>
+    <td valign="top">destdir="dir"</td>
+    <td valign="top">
+      The directory to write the generated files to.  If not set, the files
+      are written to the directory containing the grammar file. Note that unlike 
+      JFlex's "-d" command line option, <i>destdir</i> causes the generated file to be written to 
+      {destdir}/<b>{package name}</b>. This behaviour is similar to <i>javac -d dir</i>.
+    </td>
+    <td valign="top" align="center">No</td>
+    <td></td>
+  </tr>
+  <tr>
+    <td valign="top">outdir="dir"</td>
+    <td valign="top">
+      The directory to write the generated files to.  If not set, the files
+      are written to the directory containing the grammar file. This options works 
+      exactly like JFlex's "-d" command line option, it causes the output file to
+      be written to <i>dir</i> regardless of the package name.
+    </td>
+    <td valign="top" align="center">No</td>
+    <td></td>
+  </tr>
+  <tr>
+    <td valign="top">verbose</td>
+    <td valign="top">Display generation process messages.</td>
+    <td align="center" valign="top">No</td>
+    <td align="center" valign="top">&quot;off&quot;</td>
+  </tr>
+  <tr>
+    <td valign="top">dump</td>
+    <td valign="top">Dump character classes, NFA and DFA tables.</td>
+    <td align="center" valign="top">No</td>
+    <td align="center" valign="top">&quot;off&quot;</td>
+  </tr>
+  <tr>
+    <td valign="top">time or<p>timeStatistics</td>
+    <td valign="top">Display generation time statistics.</td>
+    <td align="center" valign="top">No</td>
+    <td align="center" valign="top">&quot;off&quot;</td>
+  </tr>
+  <tr>
+    <td valign="top">nomin or<p>skipMinimization</td>
+    <td valign="top">Skip minimization step.</td>
+    <td align="center" valign="top">No</td>
+    <td align="center" valign="top">&quot;off&quot;</td>
+  </tr>
+  <tr>
+    <td valign="top">skel="file" or<p>skeleton="file"</td>
+    <td valign="top">Use external skeleton file.</td>
+    <td align="center" valign="top">No</td>
+    <td align="center" valign="top">&quot;off&quot;</td>
+  </tr>
+  <tr>
+    <td valign="top">dot or<p>generateDot</td>
+    <td valign="top">Write graphviz .dot files for the generated automata (alpha).</td>
+    <td align="center" valign="top">No</td>
+    <td align="center" valign="top">&quot;off&quot;</td>
+  </tr>
+  <tr>
+    <td valign="top">nobak</td>
+    <td valign="top">Do not make a backup if the generated file exists.</td>
+    <td align="center" valign="top">No</td>
+    <td align="center" valign="top">&quot;off&quot;</td>
+  </tr>
+  <tr>
+    <td valign="top">switch</td>
+    <td valign="top">Use code generation method <i>switch</t>.</td>
+    <td align="center" valign="top">No</td>
+    <td align="center" valign="top">&quot;off&quot;</td>
+  </tr>
+  <tr>
+    <td valign="top">table</td>
+    <td valign="top">Use code generation method <i>table</t>.</td>
+    <td align="center" valign="top">No</td>
+    <td align="center" valign="top">&quot;off&quot;</td>
+  </tr>
+  <tr>
+    <td valign="top">pack</td>
+    <td valign="top">Use code generation method <i>pack</t>.</td>
+    <td align="center" valign="top">No</td>
+    <td align="center" valign="top">&quot;on&quot;</td>
+  </tr>
+
+</table>
+<h3>Example</h3>
+<blockquote><pre>
+&lt;jflex
+    file=&quot;src/parser/Parser.flex&quot;
+    destdir=&quot;build/generated/&quot;
+/&gt;
+</pre></blockquote>
+<p>
+  JFlex generates the lexical analyzer for <tt>src/parser/Parser.flex</tt> and saves the result 
+  to <tt>build/generated/parser/</tt>, providing <tt>Parser.flex</tt> declares to be in package <tt>parser</tt>.
+</p>
+
+<blockquote><pre>
+&lt;jflex
+    file=&quot;src/parser/Parser.flex&quot;
+    destdir=&quot;build/generated/&quot;
+/&gt;
+&lt;javac
+    srcdir=&quot;build/generated/&quot;
+    destdir=&quot;build/classes/&quot;
+/&gt;
+</pre></blockquote>
+<p>
+ The same as above plus compile generated classes to <i>build/classes</i>
+</p>
+<hr>
+
+</body>
+</html>
+

diff --git a/tools/lexer/jflex-1.4/doc/logo.gif b/tools/lexer/jflex-1.4/doc/logo.gif
new file mode 100644 (file)
index 0000000..dcd04fb
Binary files /dev/null and b/tools/lexer/jflex-1.4/doc/logo.gif differ

diff --git a/tools/lexer/jflex-1.4/doc/manual.css b/tools/lexer/jflex-1.4/doc/manual.css
new file mode 100644 (file)
index 0000000..1d69acf
--- /dev/null
+++ b/tools/lexer/jflex-1.4/doc/manual.css
@@ -0,0 +1,19 @@
+body  { background-color:#FFFFFF; }
+table { background-color:#F0F0F0; }
+h1    { font-size:16pt; font-style:italic; font-weight:bold; font-family:arial,helvetica; 
+        border-top-width:medium; border-top-style:solid; padding-top:5mm; margin-top:1cm; }
+h2    { font-size:14pt; font-style:italic; font-weight:bold; font-family:arial,helvetica; 
+        border-top-width:medium; border-top-style:solid; padding-top:5mm; margin-top:1cm; }
+h3    { font-size:14pt; font-weight:bold; font-family:arial,helvetica; margin-top:1cm; }  
+h4    { font-size:12pt; font-family:arial,helvetica; }
+
+SMALL.TINY                   { font-size:xx-small }
+SMALL.SCRIPTSIZE         { font-size:xx-small }
+SMALL.FOOTNOTESIZE     { font-size:x-small  }
+SMALL.SMALL                    { font-size:small    }
+BIG.LARGE                      { font-size:large    }
+BIG.XLARGE             { font-size:x-large  }
+BIG.XXLARGE                  { font-size:xx-large }
+BIG.HUGE                       { font-size:xx-large }
+BIG.XHUGE                      { font-size:xx-large }
+

diff --git a/tools/lexer/jflex-1.4/doc/manual.html b/tools/lexer/jflex-1.4/doc/manual.html
new file mode 100644 (file)
index 0000000..26713a5
--- /dev/null
+++ b/tools/lexer/jflex-1.4/doc/manual.html
@@ -0,0 +1,3920 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+
+<!--Converted with LaTeX2HTML 99.2beta8 (1.43)
+original version by:  Nikos Drakos, CBLU, University of Leeds
+* revised and updated by:  Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+  Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>JFlex User's Manual</TITLE>
+<META NAME="description" CONTENT="JFlex User's Manual">
+<META NAME="keywords" CONTENT="manual">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<META NAME="Generator" CONTENT="LaTeX2HTML v99.2beta8">
+<META HTTP-EQUIV="Content-Style-Type" CONTENT="text/css">
+
+<LINK REL="STYLESHEET" HREF="manual.css">
+
+</HEAD>
+
+<BODY >
+
+<P>
+
+<CENTER>
+<A NAME="TOP"></a>
+<A HREF="http://www.jflex.de"><IMG SRC="logo.gif" BORDER=0 HEIGHT=223 WIDTH=577></a>
+</CENTER>
+
+<P>
+<DIV ALIGN="CENTER">
+<I><FONT SIZE="+2">The Fast Lexical Analyser Generator</FONT>
+<BR></I></DIV>
+<P></P>
+<DIV ALIGN="CENTER"></DIV>
+<P></P>
+<DIV ALIGN="CENTER"><I>Copyright &#169;1998-2004 by <A NAME="tex2html1"
+  HREF="http://www.doclsf.de">Gerwin Klein</A>
+<BR></I></DIV>
+<P><P><BR>
+<DIV ALIGN="CENTER"><I><FONT SIZE="+4"><I><B>JFlex User's Manual</B></I></FONT>
+<BR></I></DIV>
+<P><P><BR>
+<DIV ALIGN="CENTER"><I>Version 1.4, April 12, 2004
+
+</I></DIV>
+
+<P>
+<BR>
+
+<H2><A NAME="SECTION00010000000000000000">
+Contents</A>
+</H2>
+<!--Table of Contents-->
+
+<UL>
+<LI><A NAME="tex2html80"
+  HREF="manual.html">Contents</A>
+<LI><A NAME="tex2html81"
+  HREF="manual.html#SECTION00020000000000000000">Introduction</A>
+<UL>
+<LI><A NAME="tex2html82"
+  HREF="manual.html#SECTION00021000000000000000">Design goals</A>
+<LI><A NAME="tex2html83"
+  HREF="manual.html#SECTION00022000000000000000">About this manual</A>
+</UL>
+<LI><A NAME="tex2html84"
+  HREF="manual.html#SECTION00030000000000000000">Installing and Running JFlex</A>
+<UL>
+<LI><A NAME="tex2html85"
+  HREF="manual.html#SECTION00031000000000000000">Installing JFlex</A>
+<LI><A NAME="tex2html86"
+  HREF="manual.html#SECTION00032000000000000000">Running JFlex</A>
+</UL>
+<LI><A NAME="tex2html87"
+  HREF="manual.html#SECTION00040000000000000000">A simple Example: How to work with JFlex</A>
+<UL>
+<LI><A NAME="tex2html88"
+  HREF="manual.html#SECTION00041000000000000000">Code to include</A>
+<LI><A NAME="tex2html89"
+  HREF="manual.html#SECTION00042000000000000000">Options and Macros</A>
+<LI><A NAME="tex2html90"
+  HREF="manual.html#SECTION00043000000000000000">Rules and Actions</A>
+<LI><A NAME="tex2html91"
+  HREF="manual.html#SECTION00044000000000000000">How to get it going</A>
+</UL>
+<LI><A NAME="tex2html92"
+  HREF="manual.html#SECTION00050000000000000000">Lexical Specifications</A>
+<UL>
+<LI><A NAME="tex2html93"
+  HREF="manual.html#SECTION00051000000000000000">User code</A>
+<LI><A NAME="tex2html94"
+  HREF="manual.html#SECTION00052000000000000000">Options and declarations</A>
+<LI><A NAME="tex2html95"
+  HREF="manual.html#SECTION00053000000000000000">Lexical rules</A>
+</UL>
+<LI><A NAME="tex2html96"
+  HREF="manual.html#SECTION00060000000000000000">Encodings, Platforms, and Unicode</A>
+<UL>
+<LI><A NAME="tex2html97"
+  HREF="manual.html#SECTION00061000000000000000">The Problem</A>
+<LI><A NAME="tex2html98"
+  HREF="manual.html#SECTION00062000000000000000">Scanning text files</A>
+<LI><A NAME="tex2html99"
+  HREF="manual.html#SECTION00063000000000000000">Scanning binaries</A>
+</UL>
+<LI><A NAME="tex2html100"
+  HREF="manual.html#SECTION00070000000000000000">A few words on performance</A>
+<UL>
+<LI><A NAME="tex2html101"
+  HREF="manual.html#SECTION00071000000000000000">Comparison of JLex and JFlex</A>
+<LI><A NAME="tex2html102"
+  HREF="manual.html#SECTION00072000000000000000">How to write a faster specification</A>
+</UL>
+<LI><A NAME="tex2html103"
+  HREF="manual.html#SECTION00080000000000000000">Porting Issues</A>
+<UL>
+<LI><A NAME="tex2html104"
+  HREF="manual.html#SECTION00081000000000000000">Porting from JLex</A>
+<LI><A NAME="tex2html105"
+  HREF="manual.html#SECTION00082000000000000000">Porting from lex/flex</A>
+</UL>
+<LI><A NAME="tex2html106"
+  HREF="manual.html#SECTION00090000000000000000">Working together</A>
+<UL>
+<LI><A NAME="tex2html107"
+  HREF="manual.html#SECTION00091000000000000000">JFlex and CUP</A>
+<LI><A NAME="tex2html108"
+  HREF="manual.html#SECTION00092000000000000000">JFlex and BYacc/J</A>
+</UL>
+<LI><A NAME="tex2html109"
+  HREF="manual.html#SECTION000100000000000000000">Bugs and Deficiencies</A>
+<UL>
+<LI><A NAME="tex2html110"
+  HREF="manual.html#SECTION000101000000000000000">Deficiencies</A>
+<LI><A NAME="tex2html111"
+  HREF="manual.html#SECTION000102000000000000000">Bugs</A>
+</UL>
+<LI><A NAME="tex2html112"
+  HREF="manual.html#SECTION000110000000000000000">Copying and License</A>
+<LI><A NAME="tex2html113"
+  HREF="manual.html#SECTION000120000000000000000">Bibliography</A>
+</UL>
+<!--End of Table of Contents-->
+
+<H1><A NAME="SECTION00020000000000000000"></A><A NAME="Intro"></A><BR>
+Introduction
+</H1>
+JFlex is a lexical analyzer generator for Java<A NAME="tex2html2"
+  HREF="#foot32"><SUP><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="footnote.png"></SUP></A>written in Java. It is also a rewrite of the very useful tool JLex [<A
+ HREF="manual.html#JLex">3</A>] which
+was developed by Elliot Berk at Princeton University. As Vern Paxon states
+for his C/C++ tool flex [<A
+ HREF="manual.html#flex">11</A>]: They do not share any code though.
+
+<P>
+
+<H2><A NAME="SECTION00021000000000000000">
+Design goals</A>
+</H2>
+The main design goals of JFlex are:
+
+<UL>
+<LI><B>Full unicode support</B>
+</LI>
+<LI><B>Fast generated scanners </B>
+</LI>
+<LI><B>Fast scanner generation</B>
+</LI>
+<LI><B>Convenient specification syntax</B>
+</LI>
+<LI><B>Platform independence</B>
+</LI>
+<LI><B>JLex compatibility</B>
+</LI>
+</UL>
+
+<P>
+
+<H2><A NAME="SECTION00022000000000000000">
+About this manual</A>
+</H2>
+This manual gives a brief but complete description of the tool JFlex. It
+assumes that you are familiar with the issue of lexical analysis. The references [<A
+ HREF="manual.html#Aho">1</A>],
+[<A
+ HREF="manual.html#Appel">2</A>], and [<A
+ HREF="manual.html#Maurer">13</A>] provide a good introduction to this topic.
+
+<P>
+The next section of this manual describes <A HREF="manual.html#Installing"><I>installation procedures</I></A> 
+for JFlex. If you never worked with JLex or
+just want to compare a JLex and a JFlex scanner specification you
+should also read <A HREF="manual.html#Example"><I>Working with JFlex - an example</I></A> 
+(section <A HREF="manual.html#Example"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>).  All options and the complete
+specification syntax are presented in 
+<A HREF="manual.html#Specifications"><I>Lexical specifications</I></A> (section <A HREF="manual.html#Specifications"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>);
+<A HREF="manual.html#sec:encodings"><I>Encodings, Platforms, and Unicode</I></A> (section <A HREF="manual.html#sec:encodings"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>)
+provides information about scannig text vs.&nbsp;binary files.
+If you are interested in performance
+considerations and comparing JLex with JFlex speed, 
+<A HREF="manual.html#performance"><I>a few words on performance</I></A> (section <A HREF="manual.html#performance"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>)
+might be just right for you. Those who want to
+use their old JLex specifications may want to check out section <A HREF="manual.html#Porting"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>
+<A HREF="manual.html#Porting"><I>Porting from JLex</I></A> to avoid possible problems
+with not portable or non standard JLex behavior that has been fixed in
+JFlex. Section <A HREF="manual.html#lexport"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A> talks about porting scanners from the
+Unix tools lex and flex.  Interfacing JFlex scanners with the LALR
+parser generators CUP and BYacc/J is explained in <A HREF="manual.html#WorkingTog"><I>working
+  together</I></A> (section <A HREF="manual.html#WorkingTog"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>).  Section <A HREF="manual.html#Bugs"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>
+<A HREF="manual.html#Bugs"><I>Bugs</I></A> gives a list of currently known active bugs. 
+The manual concludes with notes about 
+<A HREF="manual.html#Copyright"><I>Copying and License</I></A> (section <A HREF="manual.html#Copyright"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>) and
+<A HREF="manual.html#References">references</A>.
+
+<P>
+
+<H1><A NAME="SECTION00030000000000000000"></A><A NAME="Installing"></A><BR>
+Installing and Running JFlex
+</H1>
+
+<P>
+
+<H2><A NAME="SECTION00031000000000000000">
+Installing JFlex</A>
+</H2>
+
+<P>
+
+<H3><A NAME="SECTION00031100000000000000"></A><A NAME="install:windows"></A><BR>
+Windows
+</H3>
+To install JFlex on Windows 95/98/NT/XP, follow these three steps:
+
+<OL>
+<LI>Unzip the file you downloaded into the directory you want JFlex in (using
+something like 
+<A NAME="tex2html3"
+  HREF="http://www.winzip.com">WinZip</A>).
+If you unzipped it to say <code>C:\</code>, the following directory structure
+should be generated:
+
+<PRE>
+C:\JFlex\                          
+      +--bin\                   (start scripts) 
+      +--doc\                   (FAQ and manual)
+      +--examples\ 
+               +--binary\       (scanning binary files)
+               +--byaccj\       (calculator example for BYacc/J)
+               +--cup\          (calculator example for cup)
+               +--interpreter\  (interpreter example for cup)
+               +--java\         (Java lexer specification) 
+               +--simple\       (example scanner)
+               +--standalone\   (a simple standalone scanner) 
+      +--lib\                   (the precompiled classes) 
+      +--src\ 
+          +--JFlex\             (source code of JFlex) 
+          +--JFlex\gui          (source code of JFlex UI classes)
+          +--java_cup\runtime\  (source code of cup runtime classes)
+</PRE>
+
+<P>
+</LI>
+<LI>Edit the file <B><code>bin\jflex.bat</code></B>
+(in the example it's <code>C:\JFlex\bin\jflex.bat</code>)
+such that
+
+<P>
+
+<UL>
+<LI><B><TT>JAVA_HOME</TT></B> contains the directory where your Java JDK is installed
+  (for instance <code>C:\java</code>) and
+</LI>
+<LI><B><TT>JFLEX_HOME</TT></B> the directory that contains JFlex (in the example:
+  <code>C:\JFlex</code>) 
+</LI>
+</UL>
+
+<P>
+</LI>
+<LI>Include the <code>bin\</code> directory of JFlex in your path.
+(the one that contains the start script, in the example: <code>C:\JFlex\bin</code>).
+</LI>
+</OL>
+
+<P>
+
+<H3><A NAME="SECTION00031200000000000000">
+Unix with tar archive</A>
+</H3>
+
+<P>
+To install JFlex on a Unix system, follow these two steps:
+
+<UL>
+<LI>Uncompress the archive into a directory of your choice
+  with GNU tar, for instance to <TT>/usr/share</TT>:
+
+<P>
+<TT>tar -C /usr/share -xvzf jflex-1.4.tar.gz</TT>
+
+<P>
+(The example is for site wide installation. You need to
+  be root for that. User installation works exactly the
+  same way--just choose a directory where you have write
+  permission)
+
+<P>
+</LI>
+<LI>Make a symbolic link from somewhere in your binary 
+  path to <TT>bin/jflex</TT>, for instance:
+
+<P>
+<TT>ln -s /usr/share/JFlex/bin/jflex /usr/bin/jflex</TT>
+
+<P>
+If the java interpreter is not in your binary path, you
+  need to supply its location in the script <TT>bin/jflex</TT>.
+</LI>
+</UL>
+
+<P>
+You can verify the integrity of the downloaded file with
+the MD5 checksum available on the <A NAME="tex2html4"
+  HREF="http://www.jflex.de/download.html">JFlex download page</A>. 
+If you put the checksum file in the same directory
+as the archive, you run:
+
+<P>
+<code>md5sum --check </code><TT>jflex-1.4.tar.gz.md5</TT>
+
+<P>
+It should tell you
+
+<P>
+<TT>jflex-1.4.tar.gz: OK</TT>
+
+<P>
+
+<H3><A NAME="SECTION00031300000000000000">
+Linux with RPM</A>
+</H3>
+
+<P>
+
+<UL>
+<LI>become root
+</LI>
+<LI>issue
+<BR>  <TT>rpm -U jflex-1.4-0.rpm</TT>
+</LI>
+</UL>
+
+<P>
+You can verify the integrity of the downloaded <TT>rpm</TT> file with
+
+<P>
+<code>rpm --checksig </code><TT>jflex-1.4-0.rpm</TT>
+
+<P>
+This requires my pgp public key. If you don't have it, you can use
+
+<P>
+<code>rpm --checksig --nopgp </code><TT>jflex-1.4-0.rpm</TT>
+
+<P>
+or you can get it from <A NAME="tex2html5"
+  HREF="http://www.jflex.de/public-key.asc"><TT>http://www.jflex.de/public-key.asc</TT></A>.
+
+<P>
+
+<H2><A NAME="SECTION00032000000000000000">
+Running JFlex</A>
+</H2>
+You run JFlex with:
+
+<P>
+<TT>jflex &lt;options&gt; &lt;inputfiles&gt;</TT>
+
+<P>
+It is also possible to skip the start script in <code>bin\</code>
+and include the file <code>lib\JFlex.jar</code>
+in your <TT>CLASSPATH</TT> environment variable instead.
+
+<P>
+Then you run JFlex with:
+
+<P>
+<TT>java JFlex.Main &lt;options&gt; &lt;inputfiles&gt;</TT>
+
+<P>
+The input files and options are in both cases optional. If you don't provide a file name on
+the command line, JFlex will pop up a window to ask you for one.
+
+<P>
+JFlex knows about the following options:
+
+<P>
+<DL>
+<DT></DT>
+<DD><code>-d &lt;directory&gt;</code>
+<BR>  writes the generated file to the directory <code>&lt;directory&gt;</code>
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--skel &lt;file&gt;</code>
+<BR>  uses external skeleton <code>&lt;file&gt;</code>. This is mainly for JFlex
+  maintenance and special low level customizations. Use only when you
+  know what you are doing! JFlex comes with a skeleton file in the
+  <TT>src</TT> directory that reflects exactly the internal, precompiled
+  skeleton and can be used with the <TT>-skel</TT> option.
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--nomin</code>
+<BR>  skip the DFA minimization step during scanner generation.
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--jlex</code>
+<BR>  tries even harder to comply to JLex interpretation of specs.
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--dot</code>
+<BR>  generate graphviz dot files for the NFA, DFA and minimized 
+  DFA. This feature is still in alpha status, and not
+  fully implemented yet. 
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--dump</code>
+<BR>  display transition tables of NFA, initial DFA, and minimized DFA 
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--verbose</code> or <TT>-v</TT>
+<BR>  display generation progress messages (enabled by default)
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--quiet</code> or <TT>-q</TT>
+<BR>  display error messages only (no chatter about what JFlex is
+  currently doing)
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--time</code>
+<BR>  display time statistics about the code generation process
+  (not very accurate)
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--version</code>
+<BR>  print version number 
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--info</code>
+<BR>  print system and JDK information (useful if you'd like
+  to report a problem)
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--pack</code>
+<BR>  use the %pack code generation method by default
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--table</code>
+<BR>  use the %table code generation method by default
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--switch</code>
+<BR>  use the %switch code generation method by default
+
+<P>
+</DD>
+<DT></DT>
+<DD><code>--help</code> or <TT>-h</TT>
+<BR>  print a help message explaining options and usage of JFlex.
+</DD>
+</DL>
+
+<P>
+
+<H1><A NAME="SECTION00040000000000000000"></A><A NAME="Example"></A><BR>
+A simple Example: How to work with JFlex
+</H1>
+To demonstrate what a lexical specification with JFlex looks like, this
+section presents a part of the specification for the Java language.
+The example does not describe the whole lexical structure of Java programs,
+but only a small and simplified part of it (some keywords, some operators,
+comments and only two kinds of literals). It also shows how to interface
+with the LALR parser generator CUP [<A
+ HREF="manual.html#CUP">8</A>] and therefore
+uses a class <TT>sym</TT> (generated by CUP), where integer constants for
+the terminal tokens of the CUP grammar are declared. JFlex comes with a
+directory <TT>examples</TT>, where you can find a small standalone scanner
+that doesn't need other tools like CUP to give you a running example.
+The "<TT>examples</TT>" directory also contains a <EM>complete</EM> JFlex 
+specification of the lexical structure of Java programs together with the 
+CUP parser specification for Java by 
+<A NAME="tex2html6"
+  HREF="mailto:cananian@alumni.princeton.edu">C. Scott Ananian</A>, obtained
+from the CUP [<A
+ HREF="manual.html#CUP">8</A>] website (it was modified to interface with the JFlex scanner). 
+Both specifications adhere to the Java Language Specification [<A
+ HREF="manual.html#LangSpec">7</A>].
+
+<P>
+<FONT SIZE="-1"><A NAME="CodeTop"></A></FONT><PRE>
+/* JFlex example: part of Java language lexer specification */
+import java_cup.runtime.*;
+
+/**
+ * This class is a simple example lexer.
+ */
+%%
+</PRE><FONT SIZE="-1">
+<A NAME="CodeOptions"></A></FONT><PRE>
+%class Lexer
+%unicode
+%cup
+%line
+%column
+</PRE><FONT SIZE="-1">
+<A NAME="CodeScannerCode"></A></FONT><PRE>
+%{
+  StringBuffer string = new StringBuffer();
+
+  private Symbol symbol(int type) {
+    return new Symbol(type, yyline, yycolumn);
+  }
+  private Symbol symbol(int type, Object value) {
+    return new Symbol(type, yyline, yycolumn, value);
+  }
+%}
+</PRE><FONT SIZE="-1">
+<A NAME="CodeMacros"></A></FONT><PRE>
+LineTerminator = \r|\n|\r\n
+InputCharacter = [^\r\n]
+WhiteSpace     = {LineTerminator} | [ \t\f]
+
+/* comments */
+Comment = {TraditionalComment} | {EndOfLineComment} | {DocumentationComment}
+
+TraditionalComment   = "/*" [^*] ~"*/" | "/*" "*"+ "/"
+EndOfLineComment     = "//" {InputCharacter}* {LineTerminator}
+DocumentationComment = "/**" {CommentContent} "*"+ "/"
+CommentContent       = ( [^*] | \*+ [^/*] )*
+
+Identifier = [:jletter:] [:jletterdigit:]*
+
+DecIntegerLiteral = 0 | [1-9][0-9]*
+</PRE><FONT SIZE="-1">
+<A NAME="CodeStateDecl"></A></FONT><PRE>
+%state STRING
+
+%%
+</PRE><FONT SIZE="-1">
+<A NAME="CodeRulesYYINITIAL"></A></FONT><PRE>
+/* keywords */
+&lt;YYINITIAL&gt; "abstract"           { return symbol(sym.ABSTRACT); }
+&lt;YYINITIAL&gt; "boolean"            { return symbol(sym.BOOLEAN); }
+&lt;YYINITIAL&gt; "break"              { return symbol(sym.BREAK); }
+</PRE><FONT SIZE="-1">
+<A NAME="CodeRulesBunch"></A></FONT><PRE>
+&lt;YYINITIAL&gt; {
+  /* identifiers */ 
+  {Identifier}                   { return symbol(sym.IDENTIFIER); }
+ 
+  /* literals */
+  {DecIntegerLiteral}            { return symbol(sym.INTEGER_LITERAL); }
+  \"                             { string.setLength(0); yybegin(STRING); }
+
+  /* operators */
+  "="                            { return symbol(sym.EQ); }
+  "=="                           { return symbol(sym.EQEQ); }
+  "+"                            { return symbol(sym.PLUS); }
+
+  /* comments */
+  {Comment}                      { /* ignore */ }
+ 
+  /* whitespace */
+  {WhiteSpace}                   { /* ignore */ }
+}
+</PRE><FONT SIZE="-1">
+<A NAME="CodeRulesYYtext"></A></FONT><PRE>
+&lt;STRING&gt; {
+  \"                             { yybegin(YYINITIAL); 
+                                   return symbol(sym.STRING_LITERAL, 
+                                   string.toString()); }
+  [^\n\r\"\\]+                   { string.append( yytext() ); }
+  \\t                            { string.append('\t'); }
+  \\n                            { string.append('\n'); }
+
+  \\r                            { string.append('\r'); }
+  \\\"                           { string.append('\"'); }
+  \\                             { string.append('\\'); }
+}
+</PRE><FONT SIZE="-1">
+<A NAME="CodeRulesAllStates"></A></FONT><PRE>
+/* error fallback */
+.|\n                             { throw new Error("Illegal character &lt;"+
+                                                    yytext()+"&gt;"); }
+</PRE>
+
+<P>
+From this specification JFlex generates a <TT>.java</TT> file with one
+class that contains code for the scanner. The class will have a
+constructor taking a <TT>java.io.Reader</TT> from which the input is
+read. The class will also have a function <TT>yylex()</TT> that runs the
+scanner and that can be used to get the next token from the input (in this
+example the function actually has the name <TT>next_token()</TT> because
+the specification uses the <TT>%cup</TT> switch).
+
+<P>
+As with JLex, the specification consists of three parts, divided by <TT>%%</TT>:
+
+<UL>
+<LI><A HREF="manual.html#ExampleUserCode">usercode</A>,
+</LI>
+<LI><A HREF="manual.html#ExampleOptions">options and declarations</A> and 
+</LI>
+<LI><A HREF="manual.html#ExampleLexRules">lexical rules</A>.
+</LI>
+</UL>
+
+<P>
+
+<H2><A NAME="SECTION00041000000000000000"></A><A NAME="ExampleUserCode"></A><BR>
+Code to include
+</H2>
+Let's take a look at the first section, ``user code'': The text up to the
+first line starting with <TT>%%</TT> is copied verbatim to the top 
+of the generated lexer class (before the actual class declaration). 
+Beside <TT>package</TT> and <TT>import</TT> statements there is usually not much 
+to do here. If the code ends with a javadoc class comment, the generated class
+will get this comment, if not, JFlex will generate one automatically.
+
+<P>
+
+<H2><A NAME="SECTION00042000000000000000"></A><A NAME="ExampleOptions"></A><BR>
+Options and Macros
+</H2>
+The second section ``options and declarations'' is more interesting. It consists
+of a set of options, code that is included inside the generated scanner
+class, lexical states and macro declarations. Each JFlex option must begin
+a line of the specification and starts with a <TT>%</TT>. In our example
+the following options are used:
+
+<P>
+
+<UL>
+<LI><TT><A HREF="manual.html#CodeOptions">%class Lexer</A></TT> tells JFlex to give the
+  generated class the name ``Lexer'' and to write the code to a file ``<TT>Lexer.java</TT>''.
+
+<P>
+</LI>
+<LI><TT><A HREF="manual.html#CodeOptions">%unicode</A></TT> defines the set of characters the scanner will 
+  work on. For scanning text files, <TT>%unicode</TT> should always be used. See also
+  section <A HREF="manual.html#sec:encodings"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A> for more information on character sets, encodings, and
+  scanning text vs. binary files.
+
+<P>
+</LI>
+<LI><TT><A HREF="manual.html#CodeOptions">%cup</A></TT> switches to CUP compatibility
+  mode to interface with a CUP generated parser.
+
+<P>
+</LI>
+<LI><TT><A HREF="manual.html#CodeOptions">%line</A></TT> switches line counting on (the
+  current line number can be accessed via the variable <TT>yyline</TT>)
+
+<P>
+</LI>
+<LI><TT><A HREF="manual.html#CodeOptions">%column</A></TT> switches column counting on
+  (current column is accessed via <TT>yycolumn</TT>)
+
+<P>
+</LI>
+</UL>
+<A NAME="ExampleScannerCode"></A>
+<P>
+The code included in <TT><A HREF="manual.html#CodeScannerCode">%{...%}</A></TT>
+is copied verbatim into the generated lexer class source.
+Here you can declare member variables and functions that are used
+inside scanner actions. In our example we declare a <TT>StringBuffer</TT> ``<TT>string</TT>''
+in which we will store parts of string literals and two helper functions
+``<TT>symbol</TT>'' that create <TT>java_cup.runtime.Symbol</TT> objects
+with position information of the current token (see section <A HREF="manual.html#CUPWork"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>
+<A HREF="manual.html#CUPWork"><I>JFlex and CUP</I></A> 
+for how to interface with the parser generator CUP). As JFlex options, both 
+<code>%{</code> and <code>\%}</code> must begin a line. 
+<A NAME="ExampleMacros"></A>
+<P>
+The specification continues with macro declarations. Macros are
+abbreviations for regular expressions, used to make lexical specifications
+easier to read and understand. A macro declaration
+consists of a macro identifier followed by <TT>=</TT>, then followed by
+the regular expression it represents.  This regular expression may
+itself contain macro usages. Although this allows a grammar like specification
+style, macros are still just abbreviations and not non terminals - they
+cannot be recursive or mutually recursive. Cycles in macro definitions
+are detected and reported at generation time by JFlex.
+
+<P>
+Here some of the example macros in more detail:
+
+<UL>
+<LI><TT><A HREF="manual.html#CodeMacros">LineTerminator</A></TT> stands for the regular
+  expression that matches an ASCII CR, an ASCII LF or an CR followed by LF.
+
+<P>
+</LI>
+<LI><TT><A HREF="manual.html#CodeMacros">InputCharacter</A></TT> stands for all characters
+  that are not a CR or LF.
+
+<P>
+</LI>
+<LI><TT><A HREF="manual.html#CodeMacros">TraditionalComment</A></TT> is the expression
+  that matches the string <TT>"/*"</TT> followed by a character that is not
+  a <TT>*</TT> followed by anything that matches the macro 
+  <TT><A HREF="manual.html#CodeMacros">CommentContent</A></TT>
+  followed by any number of <TT>*</TT> followed by <TT>/</TT>.
+
+<P>
+</LI>
+<LI><TT><A HREF="manual.html#CodeMacros">CommentContent</A></TT> matches zero or more
+  occurrences of any character except a <TT>*</TT> or any number of 
+  <TT>*</TT> followed by a character that is not a <TT>/</TT>
+
+<P>
+</LI>
+<LI><TT><A HREF="manual.html#CodeMacros">Identifier</A></TT> matches each string that
+  starts with a character of class <TT>jletter</TT> followed by zero or more characters
+  of class <TT>jletterdigit</TT>. <TT>jletter</TT> and <TT>jletterdigit</TT>
+  are predefined character classes.  <TT>jletter</TT> includes all characters for which
+  the Java function <TT>Character.isJavaIdentifierStart</TT> returns <TT>true</TT> and 
+  <TT>jletterdigit</TT> all characters for that <TT>Character.isJavaIdentifierPart</TT>
+  returns <TT>true</TT>. 
+</LI>
+</UL>
+<A NAME="ExampleStateDecl"></A>
+<P>
+The last part of the second section in our
+lexical specification is a lexical state declaration: 
+<TT><A HREF="manual.html#CodeStateDecl">%state STRING</A></TT>
+declares a lexical state <TT>STRING</TT> that can be
+used in the ``lexical rules'' part of the specification. A state declaration
+is a line starting with <TT>%state</TT> followed by a space or comma
+separated list of state identifiers. There can be more than one line starting
+with <TT>%state</TT>.
+
+<P>
+
+<H2><A NAME="SECTION00043000000000000000"></A><A NAME="ExampleLexRules"></A><BR>
+Rules and Actions
+</H2>
+The "lexical rules" section of a JFlex specification contains regular expressions
+and actions (Java code) that are executed when the scanner matches the
+associated regular expression. As the scanner reads its input, it keeps
+track of all regular expressions and activates the action of the expression
+that has the longest match. Our specification above for instance would with input
+"<TT>breaker</TT>" match the regular expression for <TT><A HREF="manual.html#CodeMacros">Identifier</A></TT>
+and not the keyword "<TT><A HREF="manual.html#CodeRulesYYINITIAL">break</A></TT>"
+followed by the Identifier "<TT>er</TT>", because rule <code>{Identifier}</code>
+matches more of this input at once (i.e. it matches all of it)
+than any other rule in the specification. If two regular expressions both 
+have the longest match for a certain input, the scanner chooses the action 
+of the expression that appears first in the specification. In that way, we 
+get for input "<TT>break</TT>" the keyword "<TT>break</TT>" and not an 
+Identifier "<TT>break</TT>". 
+
+<P>
+Additional to regular expression matches, one can use lexical states to
+refine a specification. A lexical state acts like a start condition.
+If the scanner is in lexical state <TT>STRING</TT>, only expressions that
+are preceded by the start condition <TT>&lt;STRING&gt;</TT> can be matched.
+A start condition of a regular expression can contain more than one lexical
+state. It is then matched when the lexer is in any of these lexical states.
+The lexical state <TT>YYINITIAL</TT> is predefined and is also the state
+in which the lexer begins scanning. If a regular expression has no start
+conditions it is matched in <EM>all</EM> lexical states.
+<A NAME="ExampleRulesStateBunch"></A>
+<P>
+Since you often have a bunch of expressions with the same start conditions, 
+JFlex allows the same abbreviation as the Unix tool <TT>flex</TT>:
+<PRE>
+&lt;STRING&gt; {
+  expr1   { action1 }
+  expr2   { action2 }
+}
+</PRE>
+means that both <TT>expr1</TT> and <TT>expr2</TT> have start condition <TT>&lt;STRING&gt;</TT>.
+<A NAME="ExampleRulesYYINITIAL"></A>
+<P>
+The first three rules in our example demonstrate the syntax of a regular 
+expression preceded by the start condition <TT>&lt;YYINITIAL&gt;</TT>.
+
+<P>
+<TT><A HREF="manual.html#CodeRulesYYINITIAL">&lt;YYINITIAL&gt; "abstract"</A><code>  {</code> return symbol(sym.ABSTRACT); <code>}</code></TT>
+
+<P>
+matches the input "<TT>abstract</TT>" only if the scanner is in its
+start state "<TT>YYINITIAL</TT>". When the string "<TT>abstract</TT>" is
+matched, the scanner function returns the CUP symbol <TT>sym.ABSTRACT</TT>.
+If an action does not return a value, the scanning process is resumed immediately 
+after executing the action.
+<A NAME="ExampleRulesBunch"></A>
+<P>
+The rules enclosed in
+
+<P>
+<TT><A HREF="manual.html#CodeRulesBunch">&lt;YYINITIAL&gt; {
+<BR>  ...
+<BR>}</A></TT>
+
+<P>
+demonstrate the abbreviated syntax and are also only matched in state <TT>YYINITIAL</TT>.
+<A NAME="ExampleRulesYYbegin"></A>
+<P>
+Of these rules, one may be of special interest:
+
+<P>
+<code>\"  { </code> <TT><A HREF="manual.html#CodeRulesBunch">string.setLength(0); yybegin(STRING);</A></TT><code> }</code>
+
+<P>
+If the scanner matches a double quote in state <TT>YYINITIAL</TT> we
+have recognized the start of a string literal. Therefore we clear our <TT>StringBuffer</TT>
+that will hold the content of this string literal and tell the scanner
+with <TT>yybegin(STRING)</TT> to switch into the lexical state <TT>STRING</TT>.
+Because we do not yet return a value to the parser, our scanner proceeds
+immediately.
+<A NAME="ExampleRulesYYtext"></A>
+<P>
+In lexical state <TT>STRING</TT> another
+rule demonstrates how to refer to the input that has been matched:
+
+<P>
+<code>[^\n\r\"]+  { </code> <TT><A HREF="manual.html#CodeRulesYYtext">string.append( yytext() );</A></TT><code> }</code>
+
+<P>
+The expression <code>[^\n\r\"]+</code> matches
+all characters in the input up to the next backslash (indicating an
+escape sequence such as <code>\n</code>), double quote (indicating the end
+of the string), or line terminator (which must not occur in a string literal).
+The matched region of the input is referred to with <TT><A HREF="manual.html#CodeRulesYYtext">yytext()</A></TT>
+and appended to the content of the string literal parsed so far.
+<A NAME="ExampleRuleLast"></A>
+<P>
+The last lexical rule in the example specification
+is used as an error fallback. It matches any character in any state that
+has not been matched by another rule. It doesn't conflict with any other
+rule because it has the least priority (because it's the last rule) and
+because it matches only one character (so it can't have longest match
+precedence over any other rule).
+
+<P>
+
+<H2><A NAME="SECTION00044000000000000000">
+How to get it going</A>
+</H2>
+
+<UL>
+<LI>Install JFlex (see section <A HREF="manual.html#Installing"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A> <A HREF="manual.html#Installing"><I>Installing JFlex</I></A>)
+
+<P>
+</LI>
+<LI>If you have written your specification file (or chosen one from the <TT>examples</TT>
+directory), save it (say under the name <TT>java-lang.flex</TT>).
+
+<P>
+</LI>
+<LI>Run JFlex with
+
+<P>
+<TT>jflex java-lang.flex</TT>
+
+<P>
+</LI>
+<LI>JFlex should then report some progress messages about generating the scanner
+and write the generated code to the directory of your specification file.
+
+<P>
+</LI>
+<LI>Compile the generated <TT>.java</TT> file and your own classes. (If you
+use CUP, generate your parser classes first)
+
+<P>
+</LI>
+<LI>That's it.
+</LI>
+</UL>
+
+<P>
+
+<H1><A NAME="SECTION00050000000000000000"></A><A NAME="Specifications"></A><BR>
+Lexical Specifications
+</H1>
+As shown above, a lexical specification file for JFlex consists of three
+parts divided by a single line starting with <TT>%%</TT>:
+
+<P>
+<TT><A HREF="manual.html#SpecUsercode">UserCode</A></TT>
+<BR><TT>%%</TT>
+<BR><TT><A HREF="manual.html#SpecOptions">Options and declarations</A></TT>
+<BR><TT>%%</TT>
+<BR><TT><A HREF="manual.html#LexRules">Lexical rules</A></TT>
+
+<P>
+In all parts of the specification comments of the form
+<TT>/* comment text */</TT> and the Java style end of line comments starting with <TT>//</TT>
+are permitted. JFlex comments do nest - so the number of <TT>/*</TT> and <TT>*/</TT> 
+should be balanced.
+
+<P>
+
+<H2><A NAME="SECTION00051000000000000000"></A><A NAME="SpecUsercode"></A><BR>
+User code
+</H2>
+The first part contains user code that is copied verbatim into the beginning
+of the source file of the generated lexer before the scanner class is declared. 
+As shown in the example above, this is the place to put <TT>package</TT> 
+declarations and <TT>import</TT>
+statements. It is possible, but not considered as good Java programming
+style to put own helper class (such as token classes) in this section.
+They should get their own <TT>.java</TT> file instead.
+
+<P>
+
+<H2><A NAME="SECTION00052000000000000000"></A><A NAME="SpecOptions"></A><BR>
+Options and declarations
+</H2>
+The second part of the lexical specification contains <A HREF="manual.html#SpecOptDirectives">options</A>
+to customize your generated lexer (JFlex directives and Java code to include in
+different parts of the lexer), declarations of <A HREF="manual.html#StateDecl">lexical states</A> and 
+<A HREF="manual.html#MacroDefs">macro definitions</A> for use in the third section 
+<A HREF="manual.html#LexRules">``Lexical rules''</A> of the lexical specification file.
+<A NAME="SpecOptDirectives"></A>
+<P>
+Each JFlex directive must be situated at the beginning of  a line
+and starts with the <TT>%</TT> character. Directives that have one or
+more parameters are described as follows:
+
+<P>
+<TT>%class "classname"</TT>
+
+<P>
+means that you start a line with <TT>%class</TT> followed by a space followed
+by the name of the class for the generated scanner (the double quotes are
+<I>not</I> to be entered, see the <A HREF="manual.html#CodeOptions">example specification</A> in
+section <A HREF="manual.html#CodeOptions"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>).
+
+<P>
+
+<H3><A NAME="SECTION00052100000000000000"></A><A NAME="ClassOptions"></A><BR>
+Class options and user class code
+</H3>
+These options regard name, constructor, API, and related parts of the
+generated scanner class.
+
+<UL>
+<LI><B><TT>%class "classname"</TT></B>
+
+<P>
+Tells JFlex to give the generated class the name "<TT>classname</TT>" and to
+write the generated code to a file "<TT>classname.java</TT>". If the 
+<TT>-d &lt;directory&gt;</TT> command line option is not used, the code
+will be written to the directory where the specification file resides. If
+no <TT>%class</TT> directive is present in the specification, the generated
+class will get the name "<TT>Yylex</TT>" and will be written to a file
+"<TT>Yylex.java</TT>". There should be only one <TT>%class</TT> directive
+in a specification.
+
+<P>
+</LI>
+<LI><B><TT>%implements "interface 1"[, "interface 2", ..]</TT></B>
+
+<P>
+Makes the generated class implement the specified interfaces. If more than
+one <TT>%implements</TT> directive is present, all the specified interfaces 
+will be implemented.
+
+<P>
+</LI>
+<LI><B><TT>%extends "classname"</TT></B>
+
+<P>
+Makes the generated class a subclass of the class ``<TT>classname</TT>''.
+There should be only one <TT>%extends</TT> directive in a specification.
+
+<P>
+</LI>
+<LI><B><TT>%public</TT></B>
+
+<P>
+Makes the generated class public (the class is only accessible in its
+own package by default).
+
+<P>
+</LI>
+<LI><B><TT>%final</TT></B>
+
+<P>
+Makes the generated class final.
+
+<P>
+</LI>
+<LI><B><TT>%abstract</TT></B>
+
+<P>
+Makes the generated class abstract.
+
+<P>
+</LI>
+<LI><B><TT>%apiprivate</TT></B>
+
+<P>
+Makes all generated methods and fields of the class
+private. Exceptions are the constructor, user code in the
+specification, and, if <code>%cup</code> is present, the method
+<TT>next_token</TT>. All occurences of
+<TT>" public "</TT> (one space character before and after <TT>public</TT>)
+in the skeleton file are replaced by
+<TT>" private "</TT> (even if a user-specified skeleton is used).
+Access to the genarated class is expected to be mediated by user class
+code (see next switch).
+
+<P>
+</LI>
+<LI><B><code>%{</code></B>
+<BR><B><TT>...</TT></B>
+<BR><B><code>%}</code></B>
+
+<P>
+The code enclosed in <code>%{</code> and <code>%}</code> is copied verbatim
+into the generated class. Here you can define your own member variables
+and functions in the generated scanner. Like all options, both <code>%{</code>
+and <code>%}</code> must start a line in the specification. If more than one
+class code directive <code>%{...%}</code> is present, the code is concatenated
+in order of appearance in the specification.
+
+<P>
+</LI>
+<LI><B><code>%init{</code></B>
+<BR><B><TT>...</TT></B>
+<BR><B><code>%init}</code></B>
+
+<P>
+The code enclosed in <code>%init{</code> and <code>%init}</code> is copied
+verbatim into the constructor of the generated class. Here, member
+variables declared in the <code>%{...%}</code> directive can be initialized.
+If more than one initializer option is present, the code is concatenated
+in order of appearance in the specification.
+
+<P>
+</LI>
+<LI><B><code>%initthrow{</code></B>
+<BR><B><TT>"exception1"[, "exception2", ...]</TT></B>
+<BR><B><code>%initthrow}</code></B>
+
+<P>
+or (on a single line) just
+
+<P>
+<B><TT>%initthrow "exception1" [,  "exception2", ...]</TT></B>
+
+<P>
+Causes the specified exceptions to be declared in the <TT>throws</TT>
+clause of the constructor. If more than one <code>%initthrow{</code> <TT>...</TT> <code>%initthrow}</code>
+directive is present in the specification, all specified exceptions will
+be declared.
+
+<P>
+</LI>
+<LI><B><TT>%scanerror "exception"</TT></B>
+
+<P>
+Causes the generated scanner to throw an instance of the specified
+exception in case of an internal error (default is
+<TT>java.lang.Error</TT>).  Note that this exception is only for
+internal scanner errors. With usual specifications it should never
+occur (i.e.&nbsp;if there is an error fallback rule in the specification
+and only the documented scanner API is used).
+
+<P>
+</LI>
+<LI><B><TT>%buffer "size"</TT></B>
+
+<P>
+Set the initial size of the scan buffer to the specified value 
+(decimal, in bytes). The default value is 16384.
+
+<P>
+</LI>
+<LI><B><TT>%include "filename"</TT></B>
+
+<P>
+Replaces the <TT>%include</TT> verbatim by the specified file. This
+feature is still experimental. It works, but error reporting can be
+strange if a syntax error occurs on the last token in the included
+file.
+
+<P>
+</LI>
+</UL>
+
+<P>
+
+<H3><A NAME="SECTION00052200000000000000"></A><A NAME="ScanningMethod"></A><BR>
+Scanning method
+</H3>
+This section shows how the scanning method can be customized. You can redefine
+the name and return type of the method and it is possible to declare
+exceptions that may be thrown in one of the actions of the specification.
+If no return type is specified, the scanning method will be declared as
+returning values of class <TT>Yytoken</TT>.
+
+<UL>
+<LI><B><TT>%function "name"</TT></B>
+
+<P>
+Causes the scanning method to get the specified name. If no <TT>%function</TT>
+directive is present in the specification, the scanning method gets the
+name ``<TT>yylex</TT>''. This directive overrides settings of the 
+<TT><A HREF="manual.html#CupMode">%cup</A></TT> switch. Please note that the default name
+of the scanning method with the <TT><A HREF="manual.html#CupMode">%cup</A></TT> switch is
+<TT>next_token</TT>. Overriding this name might lead to the generated scanner
+being implicitly declared as <TT>abstract</TT>, because it does not provide
+the method <TT>next_token</TT> of the interface <TT>java_cup.runtime.Scanner</TT>.
+It is of course possible to provide a dummy implemention of that method
+in the class code section, if you still want to override the function name.
+
+<P>
+</LI>
+<LI><B><TT>%integer</TT></B>
+<BR><B><TT>%int</TT></B>
+
+<P>
+Both cause the scanning method to be declared as of Java type <TT>int</TT>.
+Actions in the specification can then return <TT>int</TT> values as tokens.
+The default end of file value under this setting is <TT>YYEOF</TT>, which is a <TT>public
+static final int</TT> member of the generated class.
+
+<P>
+</LI>
+<LI><B><TT>%intwrap</TT></B>
+
+<P>
+Causes the scanning method to be declared as of the Java wrapper type
+<TT>Integer</TT>. Actions in the specification can then return <TT>Integer</TT>
+values as tokens. The default end of file value under this setting is <TT>null</TT>.
+
+<P>
+</LI>
+<LI><B><TT>%type "typename"</TT></B>
+
+<P>
+Causes the scanning method to be declared as returning values of the specified type.
+Actions in the specification can then return values of <TT>typename</TT>
+as tokens. The default end of file value under this setting is <TT>null</TT>.
+If <TT>typename</TT> is not a subclass of <TT>java.lang.Object</TT>,
+you should specify another end of file value using the
+<A HREF="manual.html#eofval"><TT>%eofval{</TT> <TT>...</TT> <TT>%eofval}</TT></A>
+directive or the <A HREF="manual.html#EOFRule"><TT>&lt;&lt;EOF&gt;&gt;</TT> rule</A>. 
+The <TT>%type</TT> directive overrides settings of the 
+<TT><A HREF="manual.html#CupMode">%cup</A></TT> switch.
+
+<P>
+</LI>
+<LI><B><code>%yylexthrow{</code></B>
+<BR><B><TT>"exception1"[, "exception2", ... ]</TT></B>
+<BR><B><code>%yylexthrow}</code></B>
+
+<P>
+or (on a single line) just
+
+<P>
+<B><TT>%yylexthrow "exception1" [,  "exception2", ...]</TT></B>
+
+<P>
+The exceptions listed inside <code>%yylexthrow{</code> <TT>...</TT> <code>%yylexthrow}</code> 
+will be declared in the throws clause of the scanning method. If there is
+more than one <code>%yylexthrow{</code> <TT>...</TT> <code>%yylexthrow}</code> clause in
+the specification, all specified exceptions will be declared.
+</LI>
+</UL>
+
+<P>
+
+<H3><A NAME="SECTION00052300000000000000"></A><A NAME="EOF"></A><BR>
+The end of file
+</H3>
+There is always a default value that the scanning method will return when
+the end of file has been reached. You may however define a specific value
+to return and a specific piece of code that should be executed when the
+end of file is reached.
+
+<P>
+The default end of file values depends on the return type of the scanning method:
+
+<UL>
+<LI>For <B><TT>%integer</TT></B>, the scanning method will return the value
+<B><TT>YYEOF</TT></B>, which is a <TT>public static final int</TT> member
+of the generated class.
+
+<P>
+</LI>
+<LI>For <B><TT>%intwrap</TT></B>, 
+</LI>
+<LI>no specified type at all, or a 
+</LI>
+<LI>user defined type, declared using <B><TT>%type</TT></B>, the value is <B><TT>null</TT></B>.
+
+<P>
+</LI>
+<LI>In CUP compatibility mode, using <B><TT>%cup</TT></B>, the value is 
+
+<P>
+<B><TT>new java_cup.runtime.Symbol(sym.EOF)</TT></B>
+</LI>
+</UL>
+
+<P>
+User values and code to be executed at the end of file can be defined using these directives:
+
+<A NAME="eofval"></A><UL>
+<LI><B><code>%eofval{</code></B>
+<BR><B><TT>...</TT></B>
+<BR><B><code>%eofval}</code></B>
+
+<P>
+The code included in <code>%eofval{</code> <TT>...</TT> <code>%eofval}</code> will
+be copied verbatim into the scanning method and will be executed <EM>each time</EM> 
+when the end of file is reached (this is possible when
+the scanning method is called again after the end of file has been
+reached). The code should return the value that indicates the end of
+file to the parser.  There should be only one <code>%eofval{</code> 
+<TT>...</TT> <code>%eofval}</code> clause in the specification.  
+The <code>%eofval{ ... %eofval}</code> directive overrides settings of the 
+<TT><A HREF="manual.html#CupMode">%cup</A></TT> switch and <TT><A HREF="manual.html#YaccMode">%byaccj</A></TT> switch. 
+As of version 1.2 JFlex provides
+a more readable way to specify the end of file value using the
+<A HREF="manual.html#EOFRule"><TT>&lt;&lt;EOF&gt;&gt;</TT> rule</A> (see also section <A HREF="manual.html#EOFRule"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>).
+
+<P>
+</LI>
+<LI><A NAME="eof"></A>  <B><code>%eof{</code></B>
+<BR>  <B><TT>...</TT></B>
+<BR>  <B><code>%eof}</code></B> 
+
+<P>
+The code included in <code>%{eof ... %eof}</code> will be executed
+  exactly once, when the end of file is reached. The code is included
+  inside a method <TT>void yy_do_eof()</TT> and should not return any
+  value (use <code>%eofval{...%eofval}</code> or 
+  <A HREF="manual.html#EOFRule"><TT>&lt;&lt;EOF&gt;&gt;</TT></A> for this purpose). If more than one 
+  end of file code directive is present, the code will be concatenated
+  in order of appearance in the specification.
+
+<P>
+</LI>
+<LI><B><code>%eofthrow{</code></B>
+<BR>  <B><TT>"exception1"[,"exception2", ... ]</TT></B>
+<BR>  <B><code>%eofthrow}</code></B>
+
+<P>
+or (on a single line) just
+
+<P>
+<B><TT>%eofthrow "exception1" [,  "exception2", ...]</TT></B>
+
+<P>
+The exceptions listed inside <code>%eofthrow{...%eofthrow}</code> will
+  be declared in the throws clause of the method <TT>yy_do_eof()</TT>
+  (see <A HREF="manual.html#eof"><TT>%eof</TT></A> for more on that method).
+  If there is more than one <code>%eofthrow{...%eofthrow}</code> clause
+  in the specification, all specified exceptions will be declared.
+
+<P>
+<A NAME="eofclose"></A></LI>
+<LI><B><TT>%eofclose</TT></B>
+
+<P>
+Causes JFlex to close the input stream at the end of file. The code
+  <TT>yyclose()</TT> is appended to the method <TT>yy_do_eof()</TT>
+  (together with the code specified in <code>%eof{...%eof}</code>) and
+  the exception <TT>java.io.IOException</TT> is declared in the throws
+  clause of this method (together with those of 
+  <code>%eofthrow{...%eofthrow}</code>)
+
+<P>
+</LI>
+<LI><B><TT>%eofclose false</TT></B>
+
+<P>
+Turns the effect of <TT>%eofclose</TT> off again (e.g. in case closing of
+  input stream is not wanted after <TT>%cup</TT>).
+
+<P>
+</LI>
+</UL>
+
+<P>
+
+<H3><A NAME="SECTION00052400000000000000"></A><A NAME="Standalone"></A><BR>
+Standalone scanners
+</H3>
+
+<UL>
+<LI><B><TT>%debug</TT></B>
+
+<P>
+Creates a main function in the generated class that expects the name
+of an input file on the command line and then runs the scanner on this
+input file by printing information about each returned token to the Java 
+console until the end of file is reached. The information includes:
+line number (if line counting is enabled), column (if column counting is enabled),
+the matched text, and the executed action (with line number in the specification).
+
+<P>
+</LI>
+<LI><B><TT>%standalone</TT></B>
+
+<P>
+Creates a main function in the generated class that expects the name
+of an input file on the command line and then runs the scanner on this
+input file. The values returned by the scanner are ignored, but any unmatched
+text is printed to the Java console instead (as the C/C++ tool flex does, if
+run as standalone program). To avoid having to use an extra token class, the
+scanning method will be declared as having default type <TT>int</TT>, not <TT>YYtoken</TT>
+(if there isn't any other type explicitly specified).
+This is in most cases irrelevant, but could be useful to know when making
+another scanner standalone for some purpose. You should also consider using
+the <TT>%debug</TT> directive, if you just want to be able to run the scanner
+without a parser attached for testing etc.
+
+<P>
+</LI>
+</UL>
+
+<P>
+
+<H3><A NAME="SECTION00052500000000000000"></A><A NAME="CupMode"></A><BR>
+CUP compatibility
+</H3>
+You may also want to read section <A HREF="manual.html#CUPWork"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A> <A HREF="manual.html#CUPWork"><I>JFlex and CUP</I></A>
+if you are interested in how to interface your generated
+scanner with CUP.
+
+<UL>
+<LI><B><TT>%cup</TT></B>
+
+<P>
+The <TT>%cup</TT> directive enables the CUP compatibility mode and is equivalent
+to the following set of directives:
+
+<P>
+<PRE>
+%implements java_cup.runtime.Scanner
+%function next_token
+%type java_cup.runtime.Symbol
+%eofval{
+  return new java_cup.runtime.Symbol(&lt;CUPSYM&gt;.EOF);
+%eofval}
+%eofclose
+</PRE>
+
+<P>
+The value of <TT>&lt;CUPSYM&gt;</TT> defaults to <TT>sym</TT> and can be
+changed with the <TT>%cupsym</TT> directive. In JLex compatibility
+mode (<TT>-jlex</TT> switch on the command line), <TT>%eofclose</TT>
+will not be turned on.
+
+<P>
+</LI>
+<LI><B><TT>%cupsym "classname"</TT></B>
+
+<P>
+Customizes the name of the CUP generated class/interface 
+containing the names of terminal tokens. Default is <TT>sym</TT>.
+The directive should not be used after <TT>%cup</TT>, but before.
+
+<P>
+</LI>
+<LI><B><TT>%cupdebug</TT></B>
+
+<P>
+Creates a main function in the generated class that expects the name
+of an input file on the command line and then runs the scanner on this
+input file. Prints line, column, matched text, and CUP symbol name for
+each returned token to standard out.
+
+<P>
+</LI>
+</UL>
+
+<P>
+
+<H3><A NAME="SECTION00052600000000000000"></A><A NAME="YaccMode"></A><BR>
+BYacc/J compatibility
+</H3>
+You may also want to read section <A HREF="manual.html#YaccWork"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A> <A HREF="manual.html#YaccWork"><I>JFlex and BYacc/J</I></A>
+if you are interested in how to interface your generated
+scanner with Byacc/J.
+
+<UL>
+<LI><B><TT>%byacc</TT></B>
+
+<P>
+The <TT>%byacc</TT> directive enables the BYacc/J compatibility mode and is equivalent
+to the following set of directives:
+
+<P>
+<PRE>
+%integer
+%eofval{
+  return 0;
+%eofval}
+%eofclose
+</PRE>
+
+<P>
+</LI>
+</UL>
+
+<P>
+
+<H3><A NAME="SECTION00052700000000000000"></A><A NAME="CodeGeneration"></A><BR>
+Code generation
+</H3>
+The following options define what kind of lexical analyzer code JFlex
+will produce. <TT>%pack</TT> is the default setting and will be used,
+when no code generation method is specified.
+
+<P>
+
+<UL>
+<LI><B><TT>%switch</TT></B>
+
+<P>
+With <TT>%switch</TT> JFlex will generate a scanner that has
+  the DFA hard coded into a nested switch statement. This method gives
+  a good deal of compression in terms of the size of the compiled 
+  <TT>.class</TT> file while still providing very good performance. If your
+  scanner gets to big though (say more than about 200 states)
+  performance may vastly degenerate and you should consider using one
+  of the <TT>%table</TT> or <TT>%pack</TT> directives. If your scanner
+  gets even bigger (about 300 states), the Java compiler <TT>javac</TT>
+  could produce corrupted code, that will crash when executed or will
+  give you an <TT>java.lang.VerifyError</TT> when checked by the virtual
+  machine. This is due to the size limitation of 64 KB of Java
+  methods as described in the Java Virtual Machine Specification
+  [<A
+ HREF="manual.html#MachineSpec">10</A>]. In this case you will be forced to use the 
+  <TT>%pack</TT> directive, since <TT>%switch</TT>
+  usually provides more compression of the DFA table than the 
+  <TT>%table</TT> directive.
+
+<P>
+</LI>
+<LI><B><TT>%table</TT></B>
+
+<P>
+The <TT>%table</TT> direction causes JFlex to produce a classical
+  table driven scanner that encodes its DFA table in an array.  In
+  this mode, JFlex only does a small amount of table compression (see
+  [<A
+ HREF="manual.html#ParseTable">6</A>], [<A
+ HREF="manual.html#SparseTable">12</A>], [<A
+ HREF="manual.html#Aho">1</A>] and [<A
+ HREF="manual.html#Maurer">13</A>]
+  for more details on the matter of table compression) and uses the
+  same method that JLex did up to version 1.2.1. See section <A HREF="manual.html#performance"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>
+  <A HREF="manual.html#performance">performance</A> of this manual to compare
+  these methods. The same reason as above (64 KB size limitation of
+  methods) causes the same problem, when the scanner gets too big.
+  This is, because the virtual machine treats static initializers of
+  arrays as normal methods. You will in this case again be forced to
+  use the <TT>%pack</TT> directive to avoid the problem.
+
+<P>
+</LI>
+<LI><B><TT>%pack</TT></B>
+
+<P>
+<TT>%pack</TT> causes JFlex to compress the generated DFA table and to
+  store it in one or more string literals. JFlex takes care that the
+  strings are not longer than permitted by the class file format.
+  The strings have to be unpacked when
+  the first scanner object is created and initialized.
+  After unpacking the internal access to the DFA table is exactly the
+  same as with option <TT>%table</TT> -- the only extra work to be done
+  at runtime is the unpacking process which is quite fast (not noticeable
+  in normal cases). It is in time complexity proportional to the
+  size of the expanded DFA table, and it is static,
+  i.e. it is done only once for a certain scanner class -- no matter
+  how often it is instantiated.  Again, see section
+  <A HREF="manual.html#performance"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A> <A HREF="manual.html#performance">performance</A>
+  on the performance of these scanners
+  With <TT>%pack</TT>, there should be practically no
+  limitation to the size of the scanner. <TT>%pack</TT> is the default
+  setting and will be used when no code generation method is specified.
+</LI>
+</UL>
+
+<P>
+
+<H3><A NAME="SECTION00052800000000000000"></A><A NAME="CharacterSets"></A><BR>
+Character sets
+</H3>
+
+<UL>
+<LI><B><TT>%7bit</TT></B>
+
+<P>
+Causes the generated scanner to use an 7 bit input character set (character
+codes 0-127). Because this is the default value in JLex, JFlex also defaults
+to 7 bit scanners. If an input character with a code greater than 127 is
+encountered in an input at runtime, the scanner will throw an <TT>ArrayIndexOutofBoundsException</TT>.
+Not only because of this, you should consider using the <TT>%unicode</TT> directive. 
+See also section <A HREF="manual.html#sec:encodings"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A> for information about character encodings.
+
+<P>
+</LI>
+<LI><B><TT>%full</TT></B>
+<BR><B><TT>%8bit</TT></B>
+
+<P>
+Both options cause the generated scanner to use an 8 bit input character
+set (character codes 0-255). If an input character with a code greater
+than 255 is encountered in an input at runtime, the scanner will throw
+an <TT>ArrayIndexOutofBoundsException</TT>. Note that even if your platform
+uses only one byte per character, the Unicode value of a character may
+still be greater than 255. If you are scanning text files, you should
+consider using the <TT>%unicode</TT> directive. See also section <A HREF="manual.html#sec:encodings"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>
+for more information about character encodings.
+
+<P>
+</LI>
+<LI><B><TT>%unicode</TT></B>
+<BR><B><TT>%16bit</TT></B>
+
+<P>
+Both options cause the generated scanner to use the full 16 bit Unicode input
+character set (character codes 0-65535). There will be no runtime overflow when
+using this set of input characters. <TT>%unicode</TT> does not mean that the
+scanner will read two bytes at a time. What is read and what constitutes a
+character depends on the runtime platform. See also section <A HREF="manual.html#sec:encodings"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>
+for more information about character encodings.
+
+<P>
+<A NAME="caseless"></A></LI>
+<LI><B><TT>%caseless</TT></B>
+<BR><B><TT>%ignorecase</TT></B>
+
+<P>
+This option causes JFlex to handle all characters and strings in the
+specification as if they were specified in both uppercase and lowercase form.
+This enables an easy way to specify a scanner for a language with
+case insensitive keywords. The string "<TT>break</TT>" in a specification is for instance 
+handled like the expression <TT>([bB][rR][eE][aA][kK])</TT>. The <TT>%caseless</TT>
+option does not change the matched text and does not effect character classes. So
+<TT>[a]</TT> still only matches the character <TT>a</TT> and not <TT>A</TT>, too.
+Which letters are uppercase and which lowercase letters, is defined by the Unicode standard
+and determined by JFlex with the Java methods <TT>Character.toUpperCase</TT> and 
+<TT>Character.toLowerCase</TT>. In JLex compatibility
+mode (<TT>-jlex</TT> switch on the command line), <TT>%caseless</TT>
+and <TT>%ignorecase</TT> also affect character classes.
+
+<P>
+</LI>
+</UL>
+<H3><A NAME="SECTION00052900000000000000"></A><A NAME="Counting"></A><BR>
+Line, character and column counting
+</H3>
+
+<UL>
+<LI><B><TT>%char</TT></B>
+
+<P>
+Turns character counting on. The <TT>int</TT> member variable <TT>yychar</TT>
+contains the number of characters (starting with 0) from the beginning
+of input to the beginning of the current token.
+
+<P>
+</LI>
+<LI><B><TT>%line</TT></B>
+
+<P>
+Turns line counting on. The <TT>int</TT> member variable <TT>yyline</TT>
+contains the number of lines (starting with 0) from the beginning of input
+to the beginning of the current token.
+
+<P>
+</LI>
+<LI><B><TT>%column</TT></B>
+
+<P>
+Turns column counting on. The <TT>int</TT> member variable <TT>yycolumn</TT>
+contains the number of characters (starting with 0) from the beginning
+of the current line to the beginning of the current token.
+
+<P>
+</LI>
+</UL>
+
+<P>
+
+<H3><A NAME="SECTION000521000000000000000"></A><A NAME="Obsolete"></A><BR>
+Obsolete JLex options
+</H3>
+
+<UL>
+<LI><B><TT>%notunix</TT></B>
+
+<P>
+This JLex option is obsolete in JFlex but still recognized as valid directive.
+It used to switch between Windows and Unix kind of line terminators (<code>\r\n</code>
+and <code>\n</code>) for the <TT>$</TT> operator in regular expressions. JFlex
+always recognizes both styles of platform dependent line terminators.
+
+<P>
+</LI>
+<LI><B><TT>%yyeof</TT></B>
+
+<P>
+This JLex option is obsolete in JFlex but still recognized as valid directive.
+In JLex it declares a public member constant <TT>YYEOF</TT>. JFlex declares it in any case.
+</LI>
+</UL>
+
+<P>
+
+<H3><A NAME="SECTION000521100000000000000"></A><A NAME="StateDecl"></A><BR>
+State declarations
+</H3>
+State declarations have the following from:
+
+<P>
+<TT>%s[tate] "state identifier" [, "state identifier", ... ]</TT> for inclusive or
+<BR><TT>%x[state] "state identifier" [, "state identifier", ... ]</TT> for exlusive states
+
+<P>
+There may be more than one line of state declarations, each starting with
+<TT>%state</TT> or <TT>%xstate</TT> (the first character is sufficient,
+<TT>%s</TT> and <TT>%x</TT> works, too). State identifiers are letters followed 
+by a sequence of letters, digits or underscores. State identifiers can be separated 
+by whitespace or comma.
+
+<P>
+The sequence
+
+<P>
+<TT>%state STATE1</TT>
+<BR><TT>%xstate STATE3, XYZ, STATE_10</TT>
+<BR><TT>%state ABC STATE5</TT>
+
+<P>
+declares the set of identifiers <TT>STATE1, STATE3, XYZ,
+    STATE_10, ABC, STATE5</TT> as lexical states, <TT>STATE1</TT>, <TT>ABC</TT>, <TT>STATE5</TT>
+as inclusive, and <TT>STATE3</TT>, <TT>XYZ</TT>, <TT>STATE_10</TT> as exclusive. 
+See also section
+<A HREF="manual.html#HowMatched"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A> on the way lexical states influence how the input is
+matched.
+
+<P>
+
+<H3><A NAME="SECTION000521200000000000000"></A><A NAME="MacroDefs"></A><BR>
+Macro definitions
+</H3>
+A macro definition has the form
+
+<P>
+<TT>macroidentifier = regular expression</TT>
+
+<P>
+That means, a macro definition is a macro identifier (letter followed
+by a sequence of letters, digits or underscores), that can later be
+used to reference the macro, followed by optional whitespace, followed
+by an "<TT>=</TT>", followed by optional whitespace, followed by a
+regular expression (see section <A HREF="manual.html#LexRules"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A> <A HREF="manual.html#LexRules"><I>lexical
+    rules</I></A> for more information about regular expressions).
+
+<P>
+The regular expression on the right hand side must be well formed and
+must not contain the <code>^</code>, <TT>/</TT> or <TT>$</TT> operators. <B>Differently
+to JLex, macros are not just pieces of text that are expanded by copying</B>
+- they are parsed and must be well formed.
+
+<P>
+<B>This is a feature.</B> It eliminates some very hard to find bugs in
+lexical specifications (such like not having parentheses around more
+complicated macros - which is not necessary with JFlex).  See section
+<A HREF="manual.html#Porting"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A> <A HREF="manual.html#Porting"><I>Porting from JLex</I></A> for more
+details on the problems of JLex style macros.
+
+<P>
+Since it is allowed to have macro usages in macro definitions, it is
+possible to use a grammar like notation to specify the desired lexical
+structure. Macros however remain just abbreviations of the regular expressions
+they represent. They are not non terminals of a grammar and cannot be used
+recursively in any way. JFlex detects cycles in macro definitions and reports
+them at generation time. JFlex also warns you about macros that have been
+defined but never used in the ``lexical rules'' section of the specification.
+
+<P>
+
+<H2><A NAME="SECTION00053000000000000000"></A><A NAME="LexRules"></A><BR>
+Lexical rules
+</H2>
+The ``lexical rules'' section of an JFlex specification contains a set of
+regular expressions and actions (Java code) that are executed when the
+scanner matches the associated regular expression.
+
+<P>
+
+<H3><A NAME="SECTION00053100000000000000"></A><A NAME="Grammar"></A><BR>
+Syntax
+</H3>
+The syntax of the "lexical rules" section is described by the following
+BNF grammar (terminal symbols are enclosed in 'quotes'):
+
+<P>
+<PRE>
+LexicalRules ::= Rule+ 
+Rule         ::= [StateList] ['^'] RegExp [LookAhead] Action 
+               | [StateList] '&lt;&lt;EOF&gt;&gt;' Action
+               | StateGroup 
+StateGroup   ::= StateList '{' Rule+ '}' 
+StateList    ::= '&lt;' Identifier (',' Identifier)* '&gt;' 
+LookAhead    ::= '$' | '/' RegExp
+Action       ::= '{' JavaCode '}' | '|'
+
+RegExp       ::= RegExp '|' RegExp 
+               | RegExp RegExp 
+               | '(' RegExp ')'
+               | ('!'|'~') RegExp
+               | RegExp ('*'|'+'|'?')
+               | RegExp "{" Number ["," Number] "}" 
+               | '[' ['^'] (Character|Character'-'Character)* ']' 
+               | PredefinedClass 
+               | '{' Identifier '}' 
+               | '"' StringCharacter+ '"' 
+               | Character 
+
+PredefinedClass ::= '[:jletter:]' 
+                  | '[:jletterdigit:]' 
+                  | '[:letter:]' 
+                  | '[:digit:]' 
+                  | '[:uppercase:]' 
+                  | '[:lowercase:]' 
+                  | '.'
+</PRE>
+
+<P>
+<A NAME="Terminals"></A>The grammar uses the following terminal symbols:   
+
+<UL>
+<LI><TT>JavaCode</TT>
+<BR>  a sequence of <EM><TT>BlockStatements</TT></EM> as described in the Java
+  Language Specification [<A
+ HREF="manual.html#LangSpec">7</A>], section 14.2.
+
+<P>
+</LI>
+<LI><TT>Number</TT>
+<BR>  a non negative decimal integer.
+
+<P>
+</LI>
+<LI><TT>Identifier</TT>
+<BR>  a letter <code>[a-zA-Z]</code> followed by a sequence of zero or more
+  letters, digits or underscores <code>[a-zA-Z0-9_]</code>
+
+<P>
+</LI>
+<LI><TT>Character</TT>
+<BR>  an escape sequence or any unicode character that is not one of these
+  meta characters:
+  <code>  |  (  )  {  }  [  ]  &lt; &gt;  \  .  *  +  ?  ^  $  / . " ~ !</code>
+
+<P>
+</LI>
+<LI><TT>StringCharacter</TT>
+<BR>  an escape sequence or any unicode character that is not one of these
+  meta characters:
+  <code>  \  "</code>
+
+<P>
+</LI>
+<LI>An escape sequence
+
+<P>
+
+<UL>
+<LI><code>\n</code>  <code>\r</code>  <code>\t</code>  <code>\f</code>  <code>\b</code>
+</LI>
+<LI>a <code>\x</code> followed by two hexadecimal digits <TT>[a-fA-F0-9]</TT> (denoting
+    a standard ASCII escape sequence),
+
+<P>
+</LI>
+<LI>a <code>\u</code> followed by four hexadecimal digits <TT>[a-fA-F0-9]</TT>
+    (denoting an unicode escape sequence),
+
+<P>
+</LI>
+<LI>a backslash followed by a three digit octal number from 000 to 377 (denoting
+    a standard ASCII escape sequence), or
+
+<P>
+</LI>
+<LI>a backslash followed by any other unicode character that stands for this
+    character.
+
+<P>
+</LI>
+</UL>
+
+<P>
+</LI>
+</UL>
+
+<P>
+Please note that the <code>\n</code> escape sequence stands for the ASCII
+LF character - not for the end of line. If you would like to match the
+line terminator, you should use the expression <code>\r|\n|\r\n</code> if you want
+the Java conventions, or <code>\r|\n|\r\n|\u2028|\u2029|\u000B|\u000C|\u0085</code>
+if you want to be fully Unicode compliant (see also [<A
+ HREF="manual.html#unicode_rep">5</A>]).
+
+<P>
+As of version 1.1 of JFlex the whitespace characters <TT>" "</TT>
+(space) and <code>"\t"</code> (tab) can be used to improve the readability of
+regular expressions. They will be ignored by JFlex. In character
+classes and strings however, whitespace characters keep standing for
+themselves (so the string <TT>" "</TT> still matches exactly one space
+character and <code>[ \n]</code> still matches an ASCII LF or a space
+character).
+
+<P>
+JFlex applies the following standard operator precedences in regular
+expression (from highest to lowest):
+
+<P>
+
+<UL>
+<LI>unary postfix operators (<code>'*', '+', '?', {n}, {n,m}</code>)
+
+<P>
+</LI>
+<LI>unary prefix operators (<code>'!', '~'</code>)
+
+<P>
+</LI>
+<LI>concatenation (<TT>RegExp::= RegExp Regexp</TT>)
+
+<P>
+</LI>
+<LI>union (<code>RegExp::= RegExp '|' RegExp</code>)
+</LI>
+</UL>
+
+<P>
+So the expression <code>a | abc | !cd*</code> for instance is parsed as 
+<code>(a|(abc)) | ((!c)(d*))</code>.
+
+<P>
+
+<H3><A NAME="SECTION00053200000000000000"></A><A NAME="Semantics"></A><BR>
+Semantics
+</H3>
+This section gives an informal description of which text is matched by
+a regular expression (i.e. an expression described by the <TT>RegExp</TT>
+production of the grammar presented <A HREF="manual.html#Grammar">above</A>).
+
+<P>
+A regular expression that consists solely of
+
+<UL>
+<LI>a <TT>Character</TT> matches this character.
+
+<P>
+</LI>
+<LI>a character class <code>'[' (Character|Character'-'Character)* ']'</code> matches
+  any character in that class. A <TT>Character</TT> is to be considered an
+  element of a class, if it is listed in the class or if its code lies within
+  a listed character range <TT>Character'-'Character</TT>. So <code>[a0-3\n]</code>
+  for instance matches the characters
+
+<P>
+<code>a 0 1 2 3 \n</code>
+
+<P>
+If the list of characters is empty (i.e.&nbsp;just <code>[]</code>), the expression
+  matches nothing at all (the empty set), not even the empty string. This
+  may be useful in combination with the negation operator <code>'!'</code>.
+
+<P>
+</LI>
+<LI>a negated character class <code>'[^' (Character|Character'-'Character)* ']'</code>
+  matches all characters not listed in the class. If the list of characters
+  is empty (i.e. <code>[^]</code>), the expression matches any character of the
+  input character set.
+
+<P>
+</LI>
+<LI>a string <TT>'"' StringCharacter+ '"</TT> <TT>'</TT> matches the exact
+  text enclosed in double quotes. All meta characters but <code>\</code> and
+  <TT>"</TT>  loose their special meaning inside a string. See also the
+  <A HREF="manual.html#caseless"><TT>%ignorecase</TT></A> switch.
+
+<P>
+</LI>
+<LI>a macro usage <code>'{' Identifier '}'</code> matches the input that is matched
+  by the right hand side of the macro with name "<TT>Identifier</TT>".
+
+<P>
+<A NAME="predefCharCl"></A></LI>
+<LI>a predefined character class matches any of
+  the characters in that class. There are the following predefined character
+  classes:
+
+<P>
+<TT>.</TT>  contains all characters but <code>\n</code>.
+
+<P>
+All other predefined character classes are defined in the Unicode
+  specification or the Java Language Specification and determined by
+  Java functions of class
+  <TT>java</TT>.<TT>lang</TT>.<TT>Character</TT>.
+
+<P>
+<PRE>
+[:jletter:]      isJavaIdentifierStart()
+[:jletterdigit:] isJavaIdentifierPart()
+[:letter:]       isLetter()
+[:digit:]        isDigit()
+[:uppercase:]    isUpperCase()
+[:lowercase:]    isLowerCase()
+</PRE>
+
+<P>
+They are especially useful when working with the unicode character set.
+
+<P>
+</LI>
+</UL>
+
+<P>
+If <TT>a</TT> and <TT>b</TT> are regular expressions, then
+
+<P>
+<DL COMPACT>
+<DT><TT>a | b</TT></DT>
+<DD>(union) 
+
+<P>
+is the regular expression, that matches
+            all input that is matched by <TT>a</TT> or by <TT>b</TT>.
+
+<P>
+</DD>
+<DT><TT>a b</TT></DT>
+<DD>(concatenation) 
+
+<P>
+is the regular expression,
+            that matches the input matched by <TT>a</TT> followed by the 
+            input matched by <TT>b</TT>.
+
+<P>
+</DD>
+<DT><TT>a*</TT></DT>
+<DD>(kleene closure) 
+
+<P>
+matches zero or more repetitions
+            of the input matched by <TT>a</TT>
+
+<P>
+</DD>
+<DT><TT>a+</TT></DT>
+<DD>(iteration)
+
+<P>
+is equivalent to <TT>aa*</TT>
+
+<P>
+</DD>
+<DT><TT>a?</TT></DT>
+<DD>(option)
+
+<P>
+matches the empty input or the input matched
+            by <TT>a</TT>
+
+<P>
+</DD>
+<DT><TT>!a</TT></DT>
+<DD>(negation)
+
+<P>
+matches everything but the strings matched by <TT>a</TT>. 
+          Use with care: the construction of <code>!a</code> involves
+          an additional, possibly exponential NFA to DFA transformation 
+          on the NFA for <TT>a</TT>. Note that
+          with negation and union you also have (by applying DeMorgan)
+          intersection and set difference: the intersection of 
+          <TT>a</TT> and <TT>b</TT> is <code>!(!a|!b)</code>, the expression 
+          that matches everything of <TT>a</TT> not matched by <TT>b</TT> is 
+          <code>!(!a|b)</code>
+
+<P>
+</DD>
+<DT><TT>~a</TT></DT>
+<DD>(upto)
+
+<P>
+matches everything up to (and including) the first occurrence of a text
+          matched by <TT>a</TT>. The expression <code>~a</code> is equivalent
+          to <code>!([^]* a [^]* | "") a</code>. A traditional C-style comment
+          is matched by <code>"/*" ~"*/"</code>
+
+<P>
+</DD>
+<DT><TT>a{n}</TT></DT>
+<DD>(repeat)          
+
+<P>
+is equivalent to <TT>n</TT> times the concatenation of <TT>a</TT>.
+            So <code>a{4}</code> for instance is equivalent to the expression <TT>a a a a</TT>.
+            The decimal integer <TT>n</TT> must be positive.          
+
+<P>
+</DD>
+<DT><TT>a{n,m}</TT></DT>
+<DD>is equivalent to at least <TT>n</TT> times and at most <TT>m</TT> times the 
+            concatenation of <TT>a</TT>. So <code>a{2,4}</code> for instance is equivalent 
+            to the expression <code>a a a? a?</code>. Both <TT>n</TT> and <TT>m</TT> are non
+            negative decimal integers and <TT>m</TT> must not be smaller than <TT>n</TT>.
+
+<P>
+</DD>
+<DT><TT>( a )</TT></DT>
+<DD>matches the same input as <TT>a</TT>.
+
+<P>
+</DD>
+</DL>
+
+<P>
+In a lexical rule, a regular expression <TT>r</TT> may be preceded by a
+'<code>^</code>' (the beginning of line operator). <TT>r</TT> is then
+only matched at the beginning of a line in the input. A line begins
+after each occurrence of <code>\r|\n|\r\n|\u2028|\u2029|\u000B|\u000C|\u0085</code> 
+(see also [<A
+ HREF="manual.html#unicode_rep">5</A>]) and at the beginning of input.  
+The preceding line terminator in the input is not consumed and can 
+be matched by another rule.
+
+<P>
+In a lexical rule, a regular expression <TT>r</TT> may be followed by a
+lookahead expression. A lookahead expression is either a '<TT>$</TT>'
+(the end of line operator) or a <code>'/'</code> followed by an arbitrary
+regular expression. In both cases the lookahead is not consumed and
+not included in the matched text region, but it <EM>is</EM> considered
+while determining which rule has the longest match (see also 
+<A HREF="manual.html#HowMatched"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A> <A HREF="manual.html#HowMatched"><I>How the input is matched</I></A>).
+
+<P>
+In the '<TT>$</TT>' case <TT>r</TT> is only matched at the end of a line in
+the input. The end of a line is denoted by the regular expression
+<code>\r|\n|\r\n|\u2028|\u2029|\u000B|\u000C|\u0085</code>. 
+So <code>a$</code> is equivalent to <code>a / \r|\n|\r\n|\u2028|\u2029|\u000B|\u000C|\u0085</code>.This is a bit different to the situation described in [<A
+ HREF="manual.html#unicode_rep">5</A>]:
+since in JFlex <code>$</code> is a true trailing context, the end of file
+does <B>not</B> count as end of line.
+
+<P>
+<A NAME="trailingContext"></A>
+<P>
+For arbitrary lookahead (also called <EM>trailing context</EM>) the 
+expression is matched only when followed by input that matches the
+trailing context. Unfortunately the lookahead expression is not
+really arbitrary: In a rule <TT>r1 / r2</TT>, either the text matched
+by <TT>r1</TT> must have a fixed length (e.g. if <TT>r1</TT> is a string)
+or the beginning of the trailing context <TT>r2</TT> must not match the 
+end of <TT>r1</TT>. So for example <code>"abc" / "a"|"b"</code> is ok because
+<TT>"abc"</TT> has a fixed length, <code>"a"|"ab" / "x"*</code> is ok because
+no prefix of <TT>"x"*</TT> matches a postfix of <code>"a"|"ab"</code>, but
+<code>"x"|"xy" / "yx"</code> is <EM>not</EM> possible, because the postfix <TT>"y"</TT>
+of <TT>"x"|"xy"</TT> is also a prefix of <TT>"yx"</TT>. JFlex will report
+such cases at generation time. The algorithm JFlex currently uses for matching
+trailing context expressions is the one described in [<A
+ HREF="manual.html#Aho">1</A>] (leading
+to the deficiencies mentioned above).
+
+<P>
+<A NAME="EOFRule"></A>As of version 1.2, JFlex allows lex/flex style <TT>&#171;EOF&#187;</TT> rules in
+lexical specifications. A rule
+<PRE>
+[StateList]  &lt;&lt;EOF&gt;&gt;    { some action code }
+</PRE>
+is very similar to the <A HREF="manual.html#eofval"><TT>%eofval</TT> directive</A> (section <A HREF="manual.html#eofval"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>). 
+The difference lies in the optional <TT>StateList</TT> that may precede the <TT>&#171;EOF&#187;</TT> rule. The
+action code will only be executed when the end of file is read and the
+scanner is currently in one of the lexical states listed in <TT>StateList</TT>. 
+The same <TT>StateGroup</TT> (see section <A HREF="manual.html#HowMatched"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A> 
+<A HREF="manual.html#HowMatched"><I>How the input is matched</I></A>) and precedence 
+rules as in the ``normal'' rule case apply 
+(i.e. if there is more than one <TT>&#171;EOF&#187;</TT> 
+rule for a certain lexical state, the action of the one appearing 
+earlier in the specification will be executed). <TT>&#171;EOF&#187;</TT> rules 
+override settings of the <TT>%cup</TT> and <TT>%byaccj</TT> options and 
+should not be mixed with the <TT>%eofval</TT> directive.
+
+<P>
+An <TT>Action</TT> consists either of a piece of Java code enclosed in
+curly braces or is the special <code>|</code> action. The <code>|</code> action is
+an abbreviation for the action of the following expression.
+
+<P>
+Example:
+<PRE>
+expression1   |
+expression2   |
+expression3   { some action }
+</PRE>
+is equivalent to the expanded form
+<PRE>
+expression1   { some action }
+expression2   { some action }
+expression3   { some action }
+</PRE>
+
+<P>
+They are useful when you work with trailing context expressions. The 
+expression <TT>a | (c / d) | b</TT> is not syntactically legal, but can 
+easily be expressed using the <code>|</code> action:
+<PRE>
+a       |
+c / d   |
+b       { some action }
+</PRE>
+
+<P>
+
+<H3><A NAME="SECTION00053300000000000000"></A><A NAME="HowMatched"></A><BR>
+How the input is matched
+</H3>
+When consuming its input, the scanner determines the regular expression
+that matches the longest portion of the input (longest match rule). If
+there is more than one regular expression that matches the longest portion
+of input (i.e. they all match the same input), the generated scanner chooses
+the expression that appears first in the specification. After determining
+the active regular expression, the associated action is executed. If there
+is no matching regular expression, the scanner terminates the program with
+an error message (if the <TT>%standalone</TT> directive has been used, the
+scanner prints the unmatched input to <TT>java.lang.System.out</TT> instead
+and resumes scanning). 
+
+<P>
+Lexical states can be used to further restrict the set of regular expressions
+that match the current input. 
+
+<P>
+
+<UL>
+<LI>A regular expression can only be matched when its associated set of lexical
+states includes the currently active lexical state of the scanner or if
+the set of associated lexical states is empty and the currently active lexical
+state is inclusive. Exclusive and inclusive states only differ at this point: 
+rules with an empty set of associated states.
+
+<P>
+</LI>
+<LI>The currently active lexical state of the scanner can be changed from within
+an action of a regular expression using the method <TT>yybegin()</TT>.
+
+<P>
+</LI>
+<LI>The scanner starts in the inclusive lexical state 
+<TT>YYINITIAL</TT>, which is always declared by default.
+
+<P>
+</LI>
+<LI>The set of lexical states associated with a regular expression is 
+the <TT>StateList</TT> that precedes the expression. If a rule is
+contained in one or more <TT>StateGroups</TT>, then the states of
+these are also associated with the rule, i.e.&nbsp;they accumulate over
+<TT>StateGroups</TT>.
+
+<P>
+Example:
+<PRE>
+%states A, B
+%xstates C
+%%
+expr1                   { yybegin(A); action }
+&lt;YYINITIAL, A&gt; expr2    { action }
+&lt;A&gt; {
+  expr3                 { action }
+  &lt;B,C&gt; expr4           { action }
+}
+</PRE>
+The first line declares two (inclusive) lexical states <TT>A</TT> and <TT>B</TT>,
+the second line an exclusive lexical state <TT>C</TT>.
+The default (inclusive) state <TT>YYINITIAL</TT> is always implicitly there and
+doesn't need to be declared. The rule with <TT>expr1</TT> has no
+states listed, and is thus matched in all states but the exclusive
+ones, i.e.&nbsp;<TT>A</TT>, <TT>B</TT>, and <TT>YYINITIAL</TT>. In its
+action, the scanner is switched to state <TT>A</TT>. The second rule
+<TT>expr2</TT> can only match when the scanner is in state
+<TT>YYINITIAL</TT> or <TT>A</TT>. The rule <TT>expr3</TT> can only be
+matched in state <TT>A</TT> and <TT>expr4</TT> in states <TT>A</TT>, <TT>B</TT>,
+and <TT>C</TT>.
+
+<P>
+</LI>
+<LI>Lexical states are declared and used as Java <TT>int</TT> constants in
+the generated class under the same name as they are used in the specification.
+There is no guarantee that the values of these integer constants are
+distinct. They are pointers into the generated DFA table, and if JFlex
+recognizes two states as lexically equivalent (if they are used with the
+exact same set of regular expressions), then the two constants will get
+the same value.
+
+<P>
+</LI>
+</UL>
+
+<P>
+
+<H3><A NAME="SECTION00053400000000000000">
+The generated class</A>
+</H3>
+JFlex generates exactly one file containing one class from the specification
+(unless you have declared another class in the first specification section).
+
+<P>
+The generated class contains (among other things) the DFA tables, an input buffer, 
+the lexical states of the specification, a constructor, and the scanning method
+with the user supplied actions.
+
+<P>
+The name of the class is by default <TT>Yylex</TT>, it is customizable
+with the <TT>%class</TT> directive (see also section
+<A HREF="manual.html#ClassOptions"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>). The input buffer of the lexer is connected with an
+input stream over the <TT>java.io.Reader</TT> object which is passed
+to the lexer in the generated constructor. If you want to provide your
+own constructor for the lexer, you should always call the generated
+one in it to initialize the input buffer. The input buffer should not
+be accessed directly, but only over the advertised API (see also
+section <A HREF="manual.html#ScannerMethods"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>). Its internal implementation may change
+between releases or skeleton files without notice.
+
+<P>
+The main interface to the outside world is the generated scanning
+method (default name <TT>yylex</TT>, default return type
+<TT>Yytoken</TT>). Most of its aspects are customizable (name, return
+type, declared exceptions etc., see also section
+<A HREF="manual.html#ScanningMethod"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>).  If it is called, it will consume input until
+one of the expressions in the specification is matched or an error
+occurs. If an expression is matched, the corresponding action is
+executed. It may return a value of the specified return type (in which
+case the scanning method return with this value), or if it doesn't
+return a value, the scanner resumes consuming input until the next
+expression is matched. If the end of file is reached, the scanner
+executes the EOF action, and (also upon each further call to the scanning
+method) returns the specified EOF value (see also section <A HREF="manual.html#EOF"><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="crossref.png"></A>).
+
+<P>
+
+<H3><A NAME="SECTION00053500000000000000"></A><A NAME="ScannerMethods"></A><BR>
+Scanner methods and fields accessible in actions (API)
+</H3>
+Generated methods and member fields in JFlex scanners are prefixed
+with <TT>yy</TT> to indicate that they are generated and to avoid name
+conflicts with user code copied into the class. Since user code is
+part of the same class, JFlex has no language means like the
+<TT>private</TT> modifier to indicate which members and methods are
+internal and which ones belong to the API. Instead, JFlex follows a
+naming convention: everything starting with a <TT>zz</TT> prefix like
+<TT>zzStartRead</TT> is to be considered internal and subject to
+change without notice between JFlex releases. Methods and members of
+the generated class that do not have a <TT>zz</TT> prefix like
+<TT>yycharat</TT> belong to the API that the scanner class provides to
+users in action code of the specification. They will be remain stable
+and supported between JFlex releases as long as possible.
+
+<P>
+Currently, the API consists of the following methods and member fields:
+
+<UL>
+<LI><TT>String yytext()</TT>
+<BR>  returns the matched input text region
+
+<P>
+</LI>
+<LI><TT>int yylength()</TT>
+<BR>  returns the length of the matched input text region (does not require
+  a <TT>String</TT> object to be created)
+
+<P>
+</LI>
+<LI><TT>char yycharat(int pos)</TT>
+<BR>  returns the character at position <TT>pos</TT> from the matched text.
+  It is equivalent to <TT>yytext().charAt(pos)</TT>, but faster.  <TT>  pos</TT> must be a value from <TT>0</TT> to <TT>yylength()-1</TT>.
+
+<P>
+</LI>
+<LI><TT>void yyclose()</TT>
+<BR>  closes the input stream. All subsequent calls to the scanning method will 
+  return the end of file value    
+
+<P>
+</LI>
+<LI><TT>void yyreset(java.io.Reader reader)</TT>
+<BR>  closes the current input stream, and resets the scanner to read from
+  a new input stream.  All internal variables are reset, the old input
+  stream <EM>cannot</EM> be reused (content of the internal buffer is
+  discarded and lost).  The lexical state is set to <TT>YY_INITIAL</TT>.
+
+<P>
+</LI>
+<LI><TT>void yypushStream(java.io.Reader reader)</TT>
+<BR> Stores the current input stream on a stack, and
+ reads from a new stream. Lexical state, line,
+ char, and column counting remain untouched.
+ The current input stream can be restored with
+ <TT>yypopstream</TT> (usually in an <TT>&#171;EOF&#187;</TT> action).
+
+<P>
+A typical example for this are include files in
+ style of the C preprocessor. The corresponding 
+ JFlex specification could look somewhat like this:
+<PRE>
+"#include" {FILE}  { yypushStream(new FileReader(getFile(yytext()))); }
+..
+&lt;&lt;EOF&gt;&gt;        { if (yymoreStreams()) yypopStream(); else return EOF; }
+</PRE>
+
+<P>
+This method is only available in the skeleton file
+ <TT>skeleton.nested</TT>. You can find it in the 
+ <TT>src</TT> directory of the JFlex distribution.
+
+<P>
+</LI>
+<LI><TT>void yypopStream()</TT>
+<BR>  Closes the current input stream and continues to
+  read from the one on top of the stream stack.
+
+<P>
+This method is only available in the skeleton file
+  <TT>skeleton.nested</TT>. You can find it in the 
+  <TT>src</TT> directory of the JFlex distribution.
+
+<P>
+</LI>
+<LI><TT>boolean yymoreStreams()</TT>
+<BR>  Returns true iff there are still streams for <TT>yypopStream</TT> 
+  left to read from on the stream stack.
+
+<P>
+This method is only available in the skeleton file
+  <TT>skeleton.nested</TT>. You can find it in the 
+  <TT>src</TT> directory of the JFlex distribution. 
+
+<P>
+</LI>
+<LI><TT>int yystate()</TT>
+<BR>  returns the current lexical state of the scanner.
+
+<P>
+</LI>
+<LI><TT>void yybegin(int lexicalState)</TT>
+<BR>  enters the lexical state <TT>lexicalState</TT>
+
+<P>
+</LI>
+<LI><TT>void yypushback(int number)</TT>
+<BR>  pushes <TT>number</TT> characters of the matched text back into the inputstream. 
+  They will be read again in the next call of the scanning method. 
+  The number of characters to be read again must not be greater than the length
+  of the matched text. The pushed back characters will after the call of 
+  <TT>yypushback</TT> not be included in <TT>yylength</TT> and <TT>yytext()</TT>.
+  Please note that in Java strings are unchangeable, i.e. an action code like
+  <PRE>
+    String matched = yytext();
+    yypushback(1);
+    return matched;
+</PRE>
+  will return the whole matched text, while    
+  <PRE>
+    yypushback(1);
+    return yytext();
+</PRE>
+  will return the matched text minus the last character.
+
+<P>
+</LI>
+<LI><TT>int yyline</TT>
+<BR>  contains the current line of input (starting with 0, only active with
+  the <TT><A HREF="manual.html#Counting">%line</A></TT> directive)
+
+<P>
+</LI>
+<LI><TT>int yychar</TT>
+<BR>  contains the current character count in the input (starting with 0,
+  only active with the <TT><A HREF="manual.html#Counting">%char</A></TT> directive)
+
+<P>
+</LI>
+<LI><TT>int yycolumn</TT>
+<BR>  contains the current column of the current line (starting with 0, only
+  active with the <TT><A HREF="manual.html#Counting">%column</A></TT> directive)
+
+<P>
+</LI>
+</UL>
+
+<P>
+
+<H1><A NAME="SECTION00060000000000000000"></A><A NAME="sec:encodings"></A><BR>
+Encodings, Platforms, and Unicode
+</H1>
+
+<P>
+This section tries to shed some light on the issues of Unicode and
+encodings, cross platform scanning, and how to deal with binary data.
+My thanks go to Stephen Ostermiller for his input on this topic.
+
+<P>
+
+<H2><A NAME="SECTION00061000000000000000"></A><A NAME="sec:howtoencoding"></A><BR>
+The Problem
+</H2>
+
+<P>
+Before we dive straight into details, let's take a look at what the
+problem is. The problem is Java's platform independence when you want
+to use it. For scanners the interesting part about platform
+independence is character encodings and how they are handled.
+
+<P>
+If a program reads a file from disk, it gets a stream of bytes.  In
+earlier times, when the grass was green, and the world was much
+simpler, everybody knew that the byte value 65 is, of course, an A.
+It was no problem to see which bytes meant which characters (actually
+these times never existed, but anyway).  The normal Latin alphabet
+only has 26 characters, so 7 bits or 128 distinct values should surely
+be enough to map them, even if you allow yourself the luxury of upper
+and lower case.  Nowadays, things are different. The world suddenly
+grew much larger, and all kinds of people wanted all kinds of special
+characters, just because they use them in their language and writing.
+This is were the mess starts. Since the 128 distinct values were
+already filled up with other stuff, people began to use all 8 bits of
+the byte, and extended the byte/character mappings to fit their need,
+and of course everybody did it differently. Some people for instance
+may have said ``let's use the value 213 for the German character &#228;''.  Others
+may have found that 213 should much rather mean &#233;, because they didn't need
+German and wrote French instead. As long as you use your program and
+data files only on one platform, this is no problem, as all know what
+means what, and everything gets used consistently.
+
+<P>
+Now Java comes into play, and wants to run everywhere (once written,
+that is) and now there suddenly is a problem: how do I get the same
+program to say &#228; to a certain byte when it runs in Germany and maybe &#233;
+when it runs in France? And also the other way around: when I want to
+say &#233; on the screen, which byte value should I send to the operating
+system?
+
+<P>
+Java's solution to this is to use Unicode internally.  Unicode aims to
+be a superset of all known character sets and is therefore a perfect base
+for encoding things that might get used all over the world. To make
+things work correctly, you still have to know where you are and how to
+map byte values to Unicode characters and vice versa, but the
+important thing is, that this mapping is at least possible (you can
+map Kanji characters to Unicode, but you cannot map them to ASCII or
+iso-latin-1).
+
+<P>
+
+<H2><A NAME="SECTION00062000000000000000"></A><A NAME="sec:howtotext"></A><BR>
+Scanning text files
+</H2>
+
+<P>
+Scanning text files is the standard application for scanners like
+JFlex. Therefore it should also be the most convenient one. Most times 
+it is.
+
+<P>
+The following scenario works like a breeze:
+You work on a platform X, write your lexer specification there, can
+use any obscure Unicode character in it as you like, and compile the
+program. Your users work on any platform Y (possibly but not 
+necessarily something different from X), they write their input files
+on Y and they run your program on Y. No problems.
+
+<P>
+Java does this as follows:
+If you want to read anything in Java that is supposed to contain text, 
+you use a <TT>FileReader</TT> or some <TT>InputStream</TT> together with
+an <TT>InputStreamReader</TT>. <TT>InputStreams</TT> return the raw bytes, the 
+<TT>InputStreamReader</TT> converts the bytes into Unicode characters with
+the platform's default encoding. If a text file is produced on the
+same platform, the platform's default encoding should do the mapping
+correctly. Since JFlex also uses readers and Unicode internally, this
+mechanism also works for the scanner specifications. If you write an
+<TT>A</TT> in your text editor and the editor uses the platform's encoding (say <TT>A</TT> is 65), 
+then Java translates this into the logical Unicode <TT>A</TT> internally. 
+If a user writes an <TT>A</TT> on a completely different platform (say <TT>A</TT> is 237 there),
+then Java also translates this into the logical Unicode <TT>A</TT> internally. Scanning
+is performed after that translation and both match.
+
+<P>
+Note that because of this mapping from bytes to characters, you should always 
+use the <TT>%unicode</TT> switch in you lexer specification if you want to scan
+text files. <TT>%8bit</TT> may not be enough, even if
+you know that your platform only uses one byte per character. The encoding
+Cp1252 used on many Windows machines for instance knows 256 characters, but
+the character &#180; with Cp1252 code <code>\x92</code> has the Unicode value <code>\u2019</code>, which
+is larger than 255 and which would make your scanner throw an 
+<TT>ArrayIndexOutOfBoundsException</TT> if it is encountered.
+
+<P>
+So for the usual case you don't have to do anything but use the
+<TT>%unicode</TT> switch in your lexer specification.
+
+<P>
+Things may break when you produce a text file on platform X and
+consume it on a different platform Y. Let's say you have a file
+written on a Windows PC using the encoding Cp1252. Then you move
+this file to a Linux PC with encoding ISO 8859-1 and there you want
+to run your scanner on it. Java now thinks the file is encoded
+in ISO 8859-1 (the platform's default encoding) while it really is 
+encoded in Cp1252. For most characters
+Cp1252 and ISO 8859-1 are the same, but for the byte values <code>\x80</code>
+to <code>\x9f</code> they disagree: ISO 8859-1 is undefined there. You can fix
+the problem by telling Java explicitly which encoding to use. When
+constructing the <TT>InputStreamReader</TT>, you can give the encoding
+as argument. The line
+<DIV ALIGN="CENTER">
+<TT>Reader r = new InputStreamReader(input, "Cp1252"); </TT>
+
+</DIV>
+will do the trick.
+
+<P>
+Of course the encoding to use can also come from the data itself:
+for instance, when you scan a HTML page, it may have embedded 
+information about its character encoding in the headers.
+
+<P>
+More information about encodings, which ones are supported, how
+they are called, and how to set them may be found in the
+official Java documentation in the chapter about 
+internationalization. 
+The link 
+<A NAME="tex2html7"
+  HREF="http://java.sun.com/j2se/1.3/docs/guide/intl/"><TT>http://java.sun.com/j2se/1.3/docs/guide/intl/</TT></A>
+leads to an online version of this for Sun's JDK 1.3.
+
+<P>
+
+<H2><A NAME="SECTION00063000000000000000"></A><A NAME="sec:howtobinary"></A><BR>
+Scanning binaries
+</H2>
+
+<P>
+Scanning binaries is both easier and more difficult
+than scanning text files. It's easier because you want
+the raw bytes and not their meaning, i.e.&nbsp;you don't want 
+any translation.
+It's more difficult because it's not so easy to get
+``no translation'' when you use Java readers. 
+
+<P>
+The problem (for binaries) is that JFlex scanners are
+designed to work on text. Therefore the interface is
+the <TT>Reader</TT> class (there is a constructor
+for <TT>InputStream</TT> instances, but it's just there
+for convenience and wraps an <TT>InputStreamReader</TT>
+around it to get characters, not bytes). 
+You can still get a binary scanner when you write 
+your own custom <TT>InputStreamReader</TT> class that
+does explicitly no translation, but just copies
+byte values to character codes instead. It sounds
+quite easy, and actually it is no big deal, but there 
+are a few little pitfalls on the way. In the scanner
+specification you can only enter positive character
+codes (for bytes that is <code>\x00</code>
+to <code>\xFF</code>). Java's <TT>byte</TT> type on the other hand
+is a signed 8 bit integer (-128 to 127), so you have to convert 
+them properly in your custom <TT>Reader</TT>. Also, you should
+take care when you write your lexer spec: if you
+use text in there, it gets interpreted by an encoding
+first, and what scanner you get as result might depend
+on which platform you run JFlex on when you generate
+the scanner (this is what you want for text, but for binaries it
+gets in the way). If you are not sure, or if the development
+platform might change, it's probably best to use character 
+code escapes in all places, since they don't change their
+meaning.
+
+<P>
+To illustrate these points, the example in <TT>examples/binary</TT> 
+contains a very small binary scanner that tries to
+detect if a file is a Java <TT>class</TT> file. For that
+purpose it looks if the file begins with the magic number <code>\xCAFEBABE</code>.
+
+<P>
+
+<H1><A NAME="SECTION00070000000000000000"></A><A NAME="performance"></A><BR>
+A few words on performance
+</H1>
+This section gives some empirical results about the speed of JFlex generated
+scanners in comparison to those generated by JLex,
+compares a JFlex scanner with a <A HREF="manual.html#PerformanceHandwritten">handwritten</A> 
+one, and presents some <A HREF="manual.html#PerformanceTips">tips</A> on how to make
+your specification produce a faster scanner.
+
+<P>
+
+<H2><A NAME="SECTION00071000000000000000"></A><A NAME="PerformanceJLex"></A><BR>
+Comparison of JLex and JFlex
+</H2>
+Scanners generated by the tool JLex are quite fast. It was however
+possible to further improve the performance of generated scanners
+using JFlex. The following table shows the results that were produced
+by the scanner specification of a small toy programming language (in
+fact the example from the JLex website). The scanner was generated
+using JLex and all three different JFlex code generation methods. Then
+it was run on a W98 system using Sun's JDK 1.3 with different sample inputs
+of that toy programming language. All test runs were made under the
+same conditions on an otherwise idle machine. 
+
+<P>
+The values presented in the table denote the time from the first call
+to the scanning method to returning the EOF value and the speedup in
+percent. The tests were run both int the mixed (HotSpot) JVM mode and
+the pure interpreted mode.  The mixed mode JVM brings
+about a factor of 10 performance improvement, the difference between
+JLex and JFlex only decreases slightly.
+
+<P>
+<TABLE CELLPADDING=3 BORDER="1" WIDTH="100%">
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">KB</TD>
+<TD ALIGN="CENTER">JVM</TD>
+<TD ALIGN="RIGHT">JLex</TD>
+<TD ALIGN="RIGHT"><FONT SIZE="-1"><TT>%switch</TT></FONT></TD>
+<TD ALIGN="RIGHT">speedup</TD>
+<TD ALIGN="RIGHT"><FONT SIZE="-1"><TT>%table</TT></FONT></TD>
+<TD ALIGN="RIGHT">speedup</TD>
+<TD ALIGN="RIGHT"><FONT SIZE="-1"><TT>%pack</TT></FONT></TD>
+<TD ALIGN="RIGHT">speedup</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">496</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">325 ms</TD>
+<TD ALIGN="RIGHT">261 ms</TD>
+<TD ALIGN="RIGHT">24.5 %</TD>
+<TD ALIGN="RIGHT">261 ms</TD>
+<TD ALIGN="RIGHT">24.5 %</TD>
+<TD ALIGN="RIGHT">261 ms</TD>
+<TD ALIGN="RIGHT">24.5 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">187</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">127 ms</TD>
+<TD ALIGN="RIGHT">98 ms</TD>
+<TD ALIGN="RIGHT">29.6 %</TD>
+<TD ALIGN="RIGHT">94 ms</TD>
+<TD ALIGN="RIGHT">35.1 %</TD>
+<TD ALIGN="RIGHT">96 ms</TD>
+<TD ALIGN="RIGHT">32.3 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">93</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">66 ms</TD>
+<TD ALIGN="RIGHT">50 ms</TD>
+<TD ALIGN="RIGHT">32.0 %</TD>
+<TD ALIGN="RIGHT">50 ms</TD>
+<TD ALIGN="RIGHT">32.0 %</TD>
+<TD ALIGN="RIGHT">48 ms</TD>
+<TD ALIGN="RIGHT">37.5 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">496</TD>
+<TD ALIGN="CENTER">interpr.</TD>
+<TD ALIGN="RIGHT">4009 ms</TD>
+<TD ALIGN="RIGHT">3025 ms</TD>
+<TD ALIGN="RIGHT">32.5 %</TD>
+<TD ALIGN="RIGHT">3258 ms</TD>
+<TD ALIGN="RIGHT">23.1 %</TD>
+<TD ALIGN="RIGHT">3231 ms</TD>
+<TD ALIGN="RIGHT">24.1 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">187</TD>
+<TD ALIGN="CENTER">interpr.</TD>
+<TD ALIGN="RIGHT">1641 ms</TD>
+<TD ALIGN="RIGHT">1155 ms</TD>
+<TD ALIGN="RIGHT">42.1 %</TD>
+<TD ALIGN="RIGHT">1245 ms</TD>
+<TD ALIGN="RIGHT">31.8 %</TD>
+<TD ALIGN="RIGHT">1234 ms</TD>
+<TD ALIGN="RIGHT">33.0 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">93</TD>
+<TD ALIGN="CENTER">interpr.</TD>
+<TD ALIGN="RIGHT">817 ms</TD>
+<TD ALIGN="RIGHT">573 ms</TD>
+<TD ALIGN="RIGHT">42.6 %</TD>
+<TD ALIGN="RIGHT">617 ms</TD>
+<TD ALIGN="RIGHT">32.4 %</TD>
+<TD ALIGN="RIGHT">613 ms</TD>
+<TD ALIGN="RIGHT">33.3 %</TD>
+</TR>
+</TABLE>
+
+<P><BR>
+
+<P>
+Since the scanning time of the lexical analyzer examined in the table
+above includes lexical actions that often need to create new object instances,
+another table shows the execution time for the same specification with empty
+lexical actions to compare the pure scanning engines.
+
+<P>
+<TABLE CELLPADDING=3 BORDER="1" WIDTH="100%">
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">KB</TD>
+<TD ALIGN="CENTER">JVM</TD>
+<TD ALIGN="RIGHT">JLex</TD>
+<TD ALIGN="RIGHT"><FONT SIZE="-1"><TT>%switch</TT></FONT></TD>
+<TD ALIGN="RIGHT">speedup</TD>
+<TD ALIGN="RIGHT"><FONT SIZE="-1"><TT>%table</TT></FONT></TD>
+<TD ALIGN="RIGHT">speedup</TD>
+<TD ALIGN="RIGHT"><FONT SIZE="-1"><TT>%pack</TT></FONT></TD>
+<TD ALIGN="RIGHT">speedup</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">496</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">204 ms</TD>
+<TD ALIGN="RIGHT">140 ms</TD>
+<TD ALIGN="RIGHT">45.7 %</TD>
+<TD ALIGN="RIGHT">138 ms</TD>
+<TD ALIGN="RIGHT">47.8 %</TD>
+<TD ALIGN="RIGHT">140 ms</TD>
+<TD ALIGN="RIGHT">45.7 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">187</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">83 ms</TD>
+<TD ALIGN="RIGHT">55 ms</TD>
+<TD ALIGN="RIGHT">50.9 %</TD>
+<TD ALIGN="RIGHT">52 ms</TD>
+<TD ALIGN="RIGHT">59.6 %</TD>
+<TD ALIGN="RIGHT">52 ms</TD>
+<TD ALIGN="RIGHT">59.6 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">93</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">41 ms</TD>
+<TD ALIGN="RIGHT">28 ms</TD>
+<TD ALIGN="RIGHT">46.4 %</TD>
+<TD ALIGN="RIGHT">26 ms</TD>
+<TD ALIGN="RIGHT">57.7 %</TD>
+<TD ALIGN="RIGHT">26 ms</TD>
+<TD ALIGN="RIGHT">57.7 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">496</TD>
+<TD ALIGN="CENTER">interpr.</TD>
+<TD ALIGN="RIGHT">2983 ms</TD>
+<TD ALIGN="RIGHT">2036 ms</TD>
+<TD ALIGN="RIGHT">46.5 %</TD>
+<TD ALIGN="RIGHT">2230 ms</TD>
+<TD ALIGN="RIGHT">33.8 %</TD>
+<TD ALIGN="RIGHT">2232 ms</TD>
+<TD ALIGN="RIGHT">33.6 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">187</TD>
+<TD ALIGN="CENTER">interpr.</TD>
+<TD ALIGN="RIGHT">1260 ms</TD>
+<TD ALIGN="RIGHT">793 ms</TD>
+<TD ALIGN="RIGHT">58.9 %</TD>
+<TD ALIGN="RIGHT">865 ms</TD>
+<TD ALIGN="RIGHT">45.7 %</TD>
+<TD ALIGN="RIGHT">867 ms</TD>
+<TD ALIGN="RIGHT">45.3 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">93</TD>
+<TD ALIGN="CENTER">interpr.</TD>
+<TD ALIGN="RIGHT">628 ms</TD>
+<TD ALIGN="RIGHT">395 ms</TD>
+<TD ALIGN="RIGHT">59.0 %</TD>
+<TD ALIGN="RIGHT">432 ms</TD>
+<TD ALIGN="RIGHT">45.4 %</TD>
+<TD ALIGN="RIGHT">432 ms</TD>
+<TD ALIGN="RIGHT">45.4 %</TD>
+</TR>
+</TABLE>
+
+<P><BR>
+
+<P>
+Execution time of single instructions depends on the platform and
+the implementation of the Java Virtual Machine the program is executed
+on. Therefore the tables above cannot be used as a reference to which
+code generation method of JFlex is the right one to choose in general.
+The following table was produced by the same lexical specification and
+the same input on a Linux system also using Sun's JDK 1.3.
+
+<P>
+With actions:
+
+<P>
+<TABLE CELLPADDING=3 BORDER="1" WIDTH="100%">
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">KB</TD>
+<TD ALIGN="CENTER">JVM</TD>
+<TD ALIGN="RIGHT">JLex</TD>
+<TD ALIGN="RIGHT"><FONT SIZE="-1"><TT>%switch</TT></FONT></TD>
+<TD ALIGN="RIGHT">speedup</TD>
+<TD ALIGN="RIGHT"><FONT SIZE="-1"><TT>%table</TT></FONT></TD>
+<TD ALIGN="RIGHT">speedup</TD>
+<TD ALIGN="RIGHT"><FONT SIZE="-1"><TT>%pack</TT></FONT></TD>
+<TD ALIGN="RIGHT">speedup</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">496</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">246 ms</TD>
+<TD ALIGN="RIGHT">203 ms</TD>
+<TD ALIGN="RIGHT">21.2 %</TD>
+<TD ALIGN="RIGHT">193 ms</TD>
+<TD ALIGN="RIGHT">27.5 %</TD>
+<TD ALIGN="RIGHT">190 ms</TD>
+<TD ALIGN="RIGHT">29.5 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">187</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">99 ms</TD>
+<TD ALIGN="RIGHT">76 ms</TD>
+<TD ALIGN="RIGHT">30.3 %</TD>
+<TD ALIGN="RIGHT">69 ms</TD>
+<TD ALIGN="RIGHT">43.5 %</TD>
+<TD ALIGN="RIGHT">70 ms</TD>
+<TD ALIGN="RIGHT">41.4 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">93</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">48 ms</TD>
+<TD ALIGN="RIGHT">36 ms</TD>
+<TD ALIGN="RIGHT">33.3 %</TD>
+<TD ALIGN="RIGHT">34 ms</TD>
+<TD ALIGN="RIGHT">41.2 %</TD>
+<TD ALIGN="RIGHT">35 ms</TD>
+<TD ALIGN="RIGHT">37.1 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">496</TD>
+<TD ALIGN="CENTER">interpr.</TD>
+<TD ALIGN="RIGHT">3251 ms</TD>
+<TD ALIGN="RIGHT">2247 ms</TD>
+<TD ALIGN="RIGHT">44.7 %</TD>
+<TD ALIGN="RIGHT">2430 ms</TD>
+<TD ALIGN="RIGHT">33.8 %</TD>
+<TD ALIGN="RIGHT">2444 ms</TD>
+<TD ALIGN="RIGHT">33.0 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">187</TD>
+<TD ALIGN="CENTER">interpr.</TD>
+<TD ALIGN="RIGHT">1320 ms</TD>
+<TD ALIGN="RIGHT">848 ms</TD>
+<TD ALIGN="RIGHT">55.7 %</TD>
+<TD ALIGN="RIGHT">958 ms</TD>
+<TD ALIGN="RIGHT">37.8 %</TD>
+<TD ALIGN="RIGHT">920 ms</TD>
+<TD ALIGN="RIGHT">43.5 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">93</TD>
+<TD ALIGN="CENTER">interpr.</TD>
+<TD ALIGN="RIGHT">658 ms</TD>
+<TD ALIGN="RIGHT">423 ms</TD>
+<TD ALIGN="RIGHT">55.6 %</TD>
+<TD ALIGN="RIGHT">456 ms</TD>
+<TD ALIGN="RIGHT">44.3 %</TD>
+<TD ALIGN="RIGHT">452 ms</TD>
+<TD ALIGN="RIGHT">45.6 %</TD>
+</TR>
+</TABLE>
+
+<P><BR>
+
+<P>
+Without actions:
+
+<P>
+<TABLE CELLPADDING=3 BORDER="1" WIDTH="100%">
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">KB</TD>
+<TD ALIGN="CENTER">JVM</TD>
+<TD ALIGN="RIGHT">JLex</TD>
+<TD ALIGN="RIGHT"><FONT SIZE="-1"><TT>%switch</TT></FONT></TD>
+<TD ALIGN="RIGHT">speedup</TD>
+<TD ALIGN="RIGHT"><FONT SIZE="-1"><TT>%table</TT></FONT></TD>
+<TD ALIGN="RIGHT">speedup</TD>
+<TD ALIGN="RIGHT"><FONT SIZE="-1"><TT>%pack</TT></FONT></TD>
+<TD ALIGN="RIGHT">speedup</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">496</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">136 ms</TD>
+<TD ALIGN="RIGHT">78 ms</TD>
+<TD ALIGN="RIGHT">74.4 %</TD>
+<TD ALIGN="RIGHT">76 ms</TD>
+<TD ALIGN="RIGHT">78.9 %</TD>
+<TD ALIGN="RIGHT">77 ms</TD>
+<TD ALIGN="RIGHT">76.6 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">187</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">59 ms</TD>
+<TD ALIGN="RIGHT">31 ms</TD>
+<TD ALIGN="RIGHT">90.3 %</TD>
+<TD ALIGN="RIGHT">48 ms</TD>
+<TD ALIGN="RIGHT">22.9 %</TD>
+<TD ALIGN="RIGHT">32 ms</TD>
+<TD ALIGN="RIGHT">84.4 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">93</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">28 ms</TD>
+<TD ALIGN="RIGHT">15 ms</TD>
+<TD ALIGN="RIGHT">86.7 %</TD>
+<TD ALIGN="RIGHT">15 ms</TD>
+<TD ALIGN="RIGHT">86.7 %</TD>
+<TD ALIGN="RIGHT">15 ms</TD>
+<TD ALIGN="RIGHT">86.7 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">496</TD>
+<TD ALIGN="CENTER">interpr.</TD>
+<TD ALIGN="RIGHT">1992 ms</TD>
+<TD ALIGN="RIGHT">1047 ms</TD>
+<TD ALIGN="RIGHT">90.3 %</TD>
+<TD ALIGN="RIGHT">1246 ms</TD>
+<TD ALIGN="RIGHT">59.9 %</TD>
+<TD ALIGN="RIGHT">1215 ms</TD>
+<TD ALIGN="RIGHT">64.0 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">187</TD>
+<TD ALIGN="CENTER">interpr.</TD>
+<TD ALIGN="RIGHT">859 ms</TD>
+<TD ALIGN="RIGHT">408 ms</TD>
+<TD ALIGN="RIGHT">110.5 %</TD>
+<TD ALIGN="RIGHT">479 ms</TD>
+<TD ALIGN="RIGHT">79.3 %</TD>
+<TD ALIGN="RIGHT">487 ms</TD>
+<TD ALIGN="RIGHT">76.4 %</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">93</TD>
+<TD ALIGN="CENTER">interpr.</TD>
+<TD ALIGN="RIGHT">435 ms</TD>
+<TD ALIGN="RIGHT">200 ms</TD>
+<TD ALIGN="RIGHT">117.5 %</TD>
+<TD ALIGN="RIGHT">237 ms</TD>
+<TD ALIGN="RIGHT">83.5 %</TD>
+<TD ALIGN="RIGHT">242 ms</TD>
+<TD ALIGN="RIGHT">79.8 %</TD>
+</TR>
+</TABLE>
+
+<P><BR>
+
+<P>
+Although all JFlex scanners were faster than those generated by JLex, 
+slight differences between JFlex code generation methods show up when compared
+to the run on the W98 system.
+<A NAME="PerformanceHandwritten"></A>
+<P>
+The following table compares a handwritten scanner for the Java language
+obtained from the website of CUP with the JFlex generated scanner for Java
+that comes with JFlex in the <TT>examples</TT> directory. They were tested
+on different <TT>.java</TT> files on a Linux machine with Sun's JDK 1.3.
+
+<P>
+<TABLE CELLPADDING=3 BORDER="1" WIDTH="100%">
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">lines</TD>
+<TD ALIGN="RIGHT">KB</TD>
+<TD ALIGN="CENTER">JVM</TD>
+<TD ALIGN="RIGHT">handwritten scanner</TD>
+<TD ALIGN="CENTER" COLSPAN=2>JFlex generated scanner</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">19050</TD>
+<TD ALIGN="RIGHT">496</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">824 ms</TD>
+<TD ALIGN="RIGHT">248 ms</TD>
+<TD ALIGN="RIGHT">235 % faster</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">6350</TD>
+<TD ALIGN="RIGHT">165</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">272 ms</TD>
+<TD ALIGN="RIGHT">84 ms</TD>
+<TD ALIGN="RIGHT">232 % faster</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">1270</TD>
+<TD ALIGN="RIGHT">33</TD>
+<TD ALIGN="CENTER">hotspot</TD>
+<TD ALIGN="RIGHT">53 ms</TD>
+<TD ALIGN="RIGHT">18 ms</TD>
+<TD ALIGN="RIGHT">194 % faster</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">19050</TD>
+<TD ALIGN="RIGHT">496</TD>
+<TD ALIGN="CENTER">interpreted</TD>
+<TD ALIGN="RIGHT">5.83 s</TD>
+<TD ALIGN="RIGHT">3.85 s</TD>
+<TD ALIGN="RIGHT">51 % faster</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">6350</TD>
+<TD ALIGN="RIGHT">165</TD>
+<TD ALIGN="CENTER">interpreted</TD>
+<TD ALIGN="RIGHT">1.95 s</TD>
+<TD ALIGN="RIGHT">1.29 s</TD>
+<TD ALIGN="RIGHT">51 % faster</TD>
+</TR>
+<TR><TD ALIGN="LEFT">&nbsp;</TD><TD ALIGN="RIGHT">1270</TD>
+<TD ALIGN="RIGHT">33</TD>
+<TD ALIGN="CENTER">interpreted</TD>
+<TD ALIGN="RIGHT">0.38 s</TD>
+<TD ALIGN="RIGHT">0.25 s</TD>
+<TD ALIGN="RIGHT">52 % faster</TD>
+</TR>
+</TABLE>
+
+<P><BR>
+
+<P>
+Although JDK 1.3 seems to speed up the handwritten scanner if compared
+to JDK 1.1 or 1.2 more than the generated one, the generated scanner is
+still up to 3.3 times as fast as the handwritten one. One example of 
+a handwritten scanner that is
+considerably slower than the equivalent generated one is surely no
+proof for all generated scanners being faster than handwritten. It is
+clearly impossible to prove something like that, since you could
+always write the generated scanner by hand. From a software
+engineering point of view however, there is no excuse for writing a
+scanner by hand since this task takes more time, is more difficult and
+therefore more error prone than writing a compact, readable and easy
+to change lexical specification. (I'd like to add, that I do <EM>not</EM>
+think, that the handwritten scanner from the CUP website used here in
+the test is stupid or badly written or anything like that. I actually
+think, Scott did a great job with it, and that for learning about
+lexers it is quite valuable to study it or even to write a similar one
+for oneself.)
+
+<P>
+
+<H2><A NAME="SECTION00072000000000000000"></A><A NAME="PerformanceTips"></A><BR>
+How to write a faster specification
+</H2>
+Although JFlex generated scanners show good performance without
+special optimizations, there are some heuristics that can make a
+lexical specification produce an even faster scanner. Those are
+(roughly in order of performance gain):
+
+<P>
+
+<UL>
+<LI>Avoid rules that require backtracking
+
+<P>
+From the C/C++ flex [<A
+ HREF="manual.html#flex">11</A>] manpage: <EM>``Getting rid
+of backtracking is messy and often may be an enormous amount of work for
+a complicated scanner.''</EM> Backtracking is introduced by the longest match
+rule and occurs for instance on this set of expressions:
+
+<P>
+<TT>  "averylongkeyword"</TT>
+<BR><TT>  .</TT>
+
+<P>
+With input <TT>"averylongjoke"</TT> the scanner has to read all charcters
+up to <TT>'j' </TT>to decide that rule <TT>.</TT> should be matched. All
+characters of <TT>"verylong"</TT> have to be read again for the next
+matching process. Backtracking can be avoided in general by adding
+error rules that match those error conditions 
+
+<P>
+<code> "av"|"ave"|"avery"|"averyl"|..</code>
+
+<P>
+While this is impractical in most scanners, there is still the
+possibility to add a ``catch all'' rule for a lengthy list of keywords
+<PRE>
+"keyword1"  { return symbol(KEYWORD1); } 
+.. 
+"keywordn"  { return symbol(KEYWORDn); }
+[a-z]+      { error("not a keyword"); }
+</PRE>
+Most programming language scanners already have a rule like this for
+some kind of variable length identifiers.
+
+<P>
+</LI>
+<LI>Avoid line and column counting
+
+<P>
+It costs multiple additional comparisons per input character and the
+  matched text has to be rescanned for counting. In most scanners it
+  is possible to do the line counting in the specification by
+  incrementing <TT>yyline</TT> each time a line terminator has been
+  matched.  Column counting could also be included in actions. This
+  will be faster, but can in some cases become quite messy.
+
+<P>
+</LI>
+<LI>Avoid lookahead expressions and the end of line operator '$'
+
+<P>
+The trailing context will first have to be read and then (because
+  it is not to be consumed) read again. 
+
+<P>
+</LI>
+<LI>Avoid the beginning of line operator '<code>^</code>'
+
+<P>
+It costs multiple additional comparisons per match. In some
+  cases one extra lookahead character is needed (when the last character read is
+  <code>\r</code> the scanner has to read one character ahead to check if
+  the next one is an <code>\n</code> or not).
+
+<P>
+</LI>
+<LI>Match as much text as possible in a rule.
+
+<P>
+One rule is matched in the innermost loop of the scanner.  After
+  each action some overhead for setting up the internal state of the
+   scanner is necessary.
+</LI>
+</UL>
+
+<P>
+Note that writing more rules in a specification does not make the generated
+scanner slower (except when you have to switch to another code generation
+method because of the larger size).
+
+<P>
+The two main rules of optimization apply also for lexical specifications:
+
+<OL>
+<LI><B>don't do it</B>
+</LI>
+<LI><B>(for experts only) don't do it yet</B>
+</LI>
+</OL>
+
+<P>
+Some of the performance tips above contradict a readable and compact
+specification style. When in doubt or when requirements are not or not
+yet fixed: don't use them - the specification can always be optimized
+in a later state of the development process.
+
+<P>
+
+<H1><A NAME="SECTION00080000000000000000">
+Porting Issues</A>
+</H1>
+
+<P>
+
+<H2><A NAME="SECTION00081000000000000000"></A><A NAME="Porting"></A><BR>
+Porting from JLex
+</H2>
+JFlex was designed to read old JLex specifications unchanged and to
+generate a scanner which behaves exactly the same as the one generated
+by JLex with the only difference of being faster.
+
+<P>
+This works as expected on all well formed JLex specifications.
+
+<P>
+Since the statement above is somewhat absolute, let's take a look at
+what ``well formed'' means here. A JLex specification is well formed, when
+it
+
+<UL>
+<LI>generates a working scanner with JLex
+
+<P>
+</LI>
+<LI>doesn't contain the unescaped characters <TT>!</TT> and <TT>~</TT>
+
+<P>
+They are operators in JFlex while JLex treats them as normal
+  input characters. You can easily port such a JLex specification
+  to JFlex by replacing every <TT>!</TT> with <code>\!</code> and every
+  <code>~</code> with <code>\~</code> in all regular expressions.
+
+<P>
+</LI>
+<LI>has only complete regular expressions surrounded by parentheses in
+  macro definitions
+
+<P>
+This may sound a bit harsh, but could otherwise be a major problem
+  - it can also help you find some disgusting bugs in your
+  specification that didn't show up in the first place. In JLex, a
+  right hand side of a macro is just a piece of text, that is copied
+  to the point where the macro is used. With this, some weird kind of
+  stuff like
+  <PRE>
+  macro1 = ("hello"
+  macro2 = {macro1})*
+</PRE>  
+  was possible (with <TT>macro2</TT> expanding to <code>("hello")*</code>).  This
+  is not allowed in JFlex and you will have to transform such
+  definitions. There are however some more subtle kinds of errors that
+  can be introduced by JLex macros. Let's consider a definition like
+  <code>macro = a|b</code>  and a usage like <code>{macro}*</code>.
+  This expands in JLex to <code>a|b*</code> and not to the probably intended
+  <code>(a|b)*</code>.
+
+<P>
+JFlex uses always the second form of expansion, since this is the natural
+  form of thinking about abbreviations for regular expressions.
+
+<P>
+Most specifications shouldn't suffer from this problem, because
+  macros often only contain (harmless) character classes like
+  <TT>alpha = [a-zA-Z]</TT> and more dangerous definitions like
+
+<P>
+<code> ident = {alpha}({alpha}|{digit})*</code>
+
+<P>
+are only used to write rules like
+
+<P>
+<code> {ident}       { .. action .. }</code>
+
+<P>
+and not more complex expressions like
+
+<P>
+<code> {ident}*      { .. action .. }</code>
+
+<P>
+where the kind of error presented above would show up.
+</LI>
+</UL>
+
+<P>
+
+<H2><A NAME="SECTION00082000000000000000"></A><A NAME="lexport"></A><BR>
+Porting from lex/flex
+</H2>
+This section tries to give an overview of activities and possible
+problems when porting a lexical specification from the C/C++ tools lex
+and flex [<A
+ HREF="manual.html#flex">11</A>] available on most Unix systems to JFlex.
+
+<P>
+Most of the C/C++ specific features are naturally not present in JFlex,
+but most ``clean'' lex/flex lexical specifications can be ported to 
+JFlex without very much work.
+
+<P>
+This section is by far not complete and is based mainly on a survey of
+the flex man page and very little personal experience.  If you do
+engage in any porting activity from lex/flex to JFlex and encounter
+problems, have better solutions for points presented here or have just
+some tips you would like to share, please do <A NAME="tex2html8"
+  HREF="mailto:lsf@jflex.de">contact me</A>. I will
+incorporate your experiences in this manual (with all due credit to you,
+of course).
+
+<P>
+
+<H3><A NAME="SECTION00082100000000000000">
+Basic structure</A>
+</H3>
+A lexical specification for flex has the following basic structure:
+<PRE>
+definitions
+%%
+rules
+%%
+user code
+</PRE>
+
+<P>
+The <TT>user code</TT> section usually contains some C code that is used
+in actions of the <TT>rules</TT> part of the specification. For JFlex most
+of this code will have to be included in the class code <code>%{..%}</code>
+directive in the <TT>options</TT> <TT>and declarations</TT> section (after 
+translating the C code to Java, of course). 
+
+<P>
+
+<H3><A NAME="SECTION00082200000000000000">
+Macros and Regular Expression Syntax</A>
+</H3>
+The <TT>definitions</TT> section of a flex specification is quite similar
+to the <TT>options and declarations</TT> part of JFlex specs.
+
+<P>
+Macro definitions in flex have the form:
+<PRE>
+&lt;identifier&gt;  &lt;expression&gt;
+</PRE>
+To port them to JFlex macros, just insert a <TT>=</TT> between <TT>&lt;identifier&gt;</TT>
+and <TT>&lt;expression&gt;</TT>.
+
+<P>
+The syntax and semantics of regular expressions in flex are pretty much the
+same as in JFlex. A little attention is needed for some escape sequences 
+present in flex (such as <code>\a</code>) that are not supported in JFlex. These
+escape sequences should be transformed into their octal or hexadecimal 
+equivalent. 
+
+<P>
+Another point are predefined character classes. Flex offers the ones directly
+supported by C, JFlex offers the ones supported by Java. These classes will
+sometimes have to be listed manually (if there is need for this feature, it
+may be implemented in a future JFlex version).
+
+<P>
+
+<H3><A NAME="SECTION00082300000000000000">
+Lexical Rules</A>
+</H3>
+Since flex is mostly Unix based, the '<code>^</code>' (beginning of line) and
+'<code>$</code>' (end of line) operators, consider the <code>\n</code> character as only line terminator. This should usually cause not much problems, but you
+should be prepared for occurrences of <code>\r</code> or <code>\r\n</code> or one of
+the characters <code>\u2028</code>, <code>\u2029</code>, <code>\u000B</code>, <code>\u000C</code>, 
+or <code>\u0085</code>. They are considered to be line terminators in Unicode and 
+therefore may not be consumed when 
+<code>^</code> or <code>$</code> is present in a rule.
+<P>
+The trailing context algorithm of flex is better than the one used in
+JFlex. Therefore lookahead expressions could cause major headaches. JFlex
+will issue an error message at generation time, if it cannot generate
+a scanner for a certain lookahead expression. (sorry, I have no more tips here
+on that yet. If anyone knows how the flex lookahead algorithm works (or any better one)
+and can be efficiently implemented, again: please <A NAME="tex2html9"
+  HREF="mailto:lsf@jflex.de">contact me</A>).
+
+<P>
+
+<H1><A NAME="SECTION00090000000000000000"></A><A NAME="WorkingTog"></A><BR>
+Working together
+</H1>
+
+<P>
+
+<H2><A NAME="SECTION00091000000000000000"></A><A NAME="CUPWork"></A><BR>
+JFlex and CUP
+</H2>
+One of the main design goals of JFlex was to make interfacing with the free
+Java parser generator CUP [<A
+ HREF="manual.html#CUP">8</A>] as easy as possibly. 
+This has been done by giving
+the <TT><A HREF="manual.html#CupMode">%cup</A></TT> directive a special meaning. An
+interface however always has two sides. This section concentrates on the
+CUP side of the story.
+
+<P>
+
+<H3><A NAME="SECTION00091100000000000000">
+CUP version 0.10j</A>
+</H3>
+Since CUP version 0.10j, this has been simplified greatly by the new 
+CUP scanner interface <TT>java_cup.runtime.Scanner</TT>. JFlex lexers now implement
+this interface automatically when then <TT><A HREF="manual.html#CupMode">%cup</A></TT>
+switch is used. There are no special <TT>parser code</TT>, <TT>init
+  code</TT> or <TT>scan with</TT> options any more that you have to provide
+in your CUP parser specification. You can just concentrate on your grammar.
+
+<P>
+If your generated Lexer has the class name <TT>Scanner</TT>, the parser
+is started from the a main program like this:
+
+<P>
+<PRE>
+...
+  try {
+    parser p = new parser(new Scanner(new FileReader(fileName)));
+    Object result = p.parse().value;
+  }
+  catch (Exception e) {
+...
+</PRE>
+
+<P>
+
+<H3><A NAME="SECTION00091200000000000000">
+Using existing JFlex/CUP specifications with CUP 0.10j</A>
+</H3>
+If you already have an existing specification and you would like to upgrade
+both JFlex and CUP to their newest version, you will probably have to adjust
+your specification.
+
+<P>
+The main difference between the <TT><A HREF="manual.html#CupMode">%cup</A></TT> switch in
+JFlex 1.2.1 and lower, and the current JFlex version is, that JFlex scanners
+now automatically implement the <TT>java_cup.runtime.Scanner</TT> interface.
+This means, that the scanning function now changes its name from <TT>yylex()</TT>
+to <TT>next_token()</TT>. 
+
+<P>
+The main difference from older CUP versions to 0.10j is, that CUP now
+has a default constructor that accepts a <TT>java_cup.runtime.Scanner</TT> 
+as argument and that uses this scanner as
+default (so no <TT>scan with</TT> code is necessary any more).
+
+<P>
+If you have an existing CUP specification, it will probably look somewhat like this:
+<PRE>
+parser code {:
+  Lexer lexer;
+
+  public parser (java.io.Reader input) {
+    lexer = new Lexer(input);
+  }
+:};
+
+scan with {: return lexer.yylex(); :};
+</PRE>
+
+<P>
+To upgrade to CUP 0.10j, you could change it to look like this:
+<PRE>
+parser code {:
+  public parser (java.io.Reader input) {
+    super(new Lexer(input));
+  }
+:};
+</PRE>
+
+<P>
+If you do not mind to change the method that is calling the parser,
+you could remove the constructor entirely (and if there is nothing else
+in it, the whole <TT>parser code</TT> section as well, of course). The calling
+main procedure would then construct the parser as shown in the section above.
+
+<P>
+The JFlex specification does not need to be changed.
+
+<P>
+
+<H3><A NAME="SECTION00091300000000000000">
+Using older versions of CUP</A>
+</H3>
+For people, who like or have to use older versions of CUP, the following section 
+explains ``the old way''. Please note, that the standard name of the scanning 
+function with the <TT><A HREF="manual.html#CupMode">%cup</A></TT> switch is not 
+<TT>yylex()</TT>, but <TT>next_token()</TT>.
+
+<P>
+If you have a scanner specification that begins like this:
+
+<P>
+<PRE>
+package PACKAGE;
+import java_cup.runtime.*;   /* this is convenience, but not necessary */
+ 
+%%
+ 
+%class Lexer
+%cup
+..
+</PRE>
+
+<P>
+then it matches a CUP specification starting  like
+
+<P>
+<PRE>
+package PACKAGE;
+
+parser code {:
+  Lexer lexer;
+
+  public parser (java.io.Reader input) {
+    lexer = new Lexer(input);
+  }
+:};
+
+scan with {: return lexer.next_token(); :};
+
+..
+</PRE>
+
+<P>
+This assumes that the generated parser will get the name <TT>parser</TT>.
+If it doesn't, you have to adjust the constructor name.
+
+<P>
+The parser can then be started in a main routine like this:
+
+<P>
+<PRE>
+..
+  try {
+    parser p = new parser(new FileReader(fileName));
+    Object result = p.parse().value; 
+  }
+  catch (Exception e) {
+..
+</PRE>
+
+<P>
+If you want the parser specification to be independent of the name of the generated
+scanner, you can instead write an interface Lexer
+
+<P>
+<PRE>
+public interface Lexer {
+  public java_cup.runtime.Symbol next_token() throws java.io.IOException;
+}
+</PRE>
+
+<P>
+change the parser code to:
+
+<P>
+<PRE>
+package PACKAGE;
+
+parser code {:
+  Lexer lexer;
+
+  public parser (Lexer lexer) {
+    this.lexer = lexer;
+  }
+:};
+
+scan with {: return lexer.next_token(); :};
+
+..
+</PRE>
+
+<P>
+tell JFlex about the Lexer 
+interface using the <TT>%implements</TT>
+directive:
+
+<P>
+<PRE>
+..
+%class Scanner     /* not Lexer now since that is our interface! */
+%implements Lexer
+%cup
+..
+</PRE>
+
+<P>
+and finally change the main routine to look like
+
+<P>
+<PRE>
+...
+  try {
+    parser p = new parser(new Scanner(new FileReader(fileName)));
+    Object result = p.parse().value;
+  }
+  catch (Exception e) {
+...
+</PRE>
+
+<P>
+If you want to improve the error messages that CUP generated parsers
+produce, you can also override the methods <TT>report_error</TT> and <TT>report_fatal_error</TT>
+in the ``parser code'' section of the CUP specification. The new methods
+could for instance use <TT>yyline</TT> and <TT>yycolumn</TT> (stored in
+the <TT>left</TT> and <TT>right</TT> members of class <TT>java_cup.runtime.Symbol</TT>)
+to report error positions more conveniently for the user. The lexer and
+parser for the Java language in the <TT>examples/java</TT> directory of the
+JFlex distribution use this style of error reporting. These specifications
+also demonstrate the techniques above in action.
+
+<P>
+
+<H2><A NAME="SECTION00092000000000000000"></A><A NAME="YaccWork"></A><BR>
+JFlex and BYacc/J
+</H2>
+
+<P>
+JFlex has builtin support for the Java extension 
+<A NAME="tex2html10"
+  HREF="http://troi.lincom-asg.com/~rjamison/byacc/">BYacc/J</A>
+[<A
+ HREF="manual.html#BYaccJ">9</A>] by Bob Jamison
+to the classical Berkeley Yacc parser generator.
+This section describes how to interface BYacc/J with JFlex. It
+builds on many helpful suggestions and comments from Larry Bell.
+
+<P>
+Since Yacc's architecture is a bit different from CUP's, the
+interface setup also works in a slightly different manner.
+BYacc/J expects a function <TT>int yylex()</TT> in the parser
+class that returns each next token. Semantic values are expected
+in a field <TT>yylval</TT> of type <TT>parserval</TT> where ``<TT>parser</TT>''
+is the name of the generated parser class.
+
+<P>
+For a small calculator example, one could use a setup like the 
+following on the JFlex side:
+
+<P>
+<PRE>
+%%
+
+%byaccj
+
+%{
+  /* store a reference to the parser object */
+  private parser yyparser;
+
+  /* constructor taking an additional parser object */
+  public Yylex(java.io.Reader r, parser yyparser) {
+    this(r);
+    this.yyparser = yyparser;
+  }
+%}
+
+NUM = [0-9]+ ("." [0-9]+)?
+NL  = \n | \r | \r\n
+
+%%
+
+/* operators */
+"+" | 
+..
+"(" | 
+")"    { return (int) yycharat(0); }
+
+/* newline */
+{NL}   { return parser.NL; }
+
+/* float */
+{NUM}  { yyparser.yylval = new parserval(Double.parseDouble(yytext()));
+         return parser.NUM; }
+</PRE>
+
+<P>
+The lexer expects a reference to the parser in its constructor.
+Since Yacc allows direct use of terminal characters like <TT>'+'</TT>
+in its specifications, we just return the character code for
+single char matches (e.g. the operators in the example). Symbolic
+token names are stored as <TT>public static int</TT> constants in
+the generated parser class. They are used as in the <TT>NL</TT> token
+above. Finally, for some tokens, a semantic value may have to be 
+communicated to the parser. The <TT>NUM</TT> rule demonstrates that
+bit.
+
+<P>
+A matching BYacc/J parser specification could look like this:
+<PRE>
+%{
+  import java.io.*;
+%}
+      
+%token NL          /* newline  */
+%token &lt;dval&gt; NUM  /* a number */
+
+%type &lt;dval&gt; exp
+
+%left '-' '+'
+..
+%right '^'         /* exponentiation */
+      
+%%
+
+..
+      
+exp:     NUM          { $$ = $1; }
+       | exp '+' exp  { $$ = $1 + $3; }
+       ..
+       | exp '^' exp  { $$ = Math.pow($1, $3); }
+       | '(' exp ')'  { $$ = $2; }
+       ;
+
+%%
+  /* a reference to the lexer object */
+  private Yylex lexer;
+
+  /* interface to the lexer */
+  private int yylex () {
+    int yyl_return = -1;
+    try {
+      yyl_return = lexer.yylex();
+    }
+    catch (IOException e) {
+      System.err.println("IO error :"+e);
+    }
+    return yyl_return;
+  }
+
+  /* error reporting */
+  public void yyerror (String error) {
+    System.err.println ("Error: " + error);
+  }
+
+  /* lexer is created in the constructor */
+  public parser(Reader r) {
+    lexer = new Yylex(r, this);
+  }
+
+  /* that's how you use the parser */
+  public static void main(String args[]) throws IOException {
+    parser yyparser = new parser(new FileReader(args[0]));
+    yyparser.yyparse();    
+  }
+</PRE>
+
+<P>
+Here, the customized part is mostly in the user code section:
+We create the lexer in the constructor of the parser and store
+a reference to it for later use in the parser's <TT>int yylex()</TT>
+method. This <TT>yylex</TT> in the parser only calls <TT>int yylex()</TT>
+of the generated lexer and passes the result on. If something goes
+wrong, it returns -1 to indicate an error.
+
+<P>
+Runnable versions of the specifications above 
+are located in the <TT>examples/byaccj</TT> directory of the JFlex 
+distribution.
+
+<P>
+
+<H1><A NAME="SECTION000100000000000000000"></A><A NAME="Bugs"></A><BR>
+Bugs and Deficiencies
+</H1>
+
+<P>
+
+<H2><A NAME="SECTION000101000000000000000">
+Deficiencies</A>
+</H2>
+The trailing context algorithm described in [<A
+ HREF="manual.html#Aho">1</A>] and used in
+JFlex is incorrect. It does not work, when a postfix of the regular
+expression matches a prefix of the trailing context and the length
+of the text matched by the expression does not have a fixed size.
+JFlex will report these cases as errors at generation time.
+
+<P>
+
+<H2><A NAME="SECTION000102000000000000000">
+Bugs</A>
+</H2>
+
+<P>
+As of April 12, 2004 the following bugs are known in JFlex:
+
+<UL>
+<LI>The check if a lookahead expression is legal fails on some expressions.
+  The lookahead algorithm itself works as advertised, but JFlex will not
+  report all lookahead expressions that the algorithm can't handle at generation
+  time. Some cases are caught by the check, but not all.
+
+<P>
+<B>Workaround:</B> Check lookahead expressions manually. A lookahead expression 
+  <TT>r1/r2</TT> is ok, if no postfix of <TT>r1</TT> can match a prefix of <TT>r2</TT>.
+</LI>
+</UL>
+
+<P>
+If you find new ones, please use the bugs section of the
+<A NAME="tex2html11"
+  HREF="http://www.jflex.de/">JFlex website</A>
+to report them.
+
+<P>
+
+<H1><A NAME="SECTION000110000000000000000"></A><A NAME="Copyright"></A><BR>
+Copying and License
+</H1>
+JFlex is free software, published under the terms of the 
+<A NAME="tex2html12"
+  HREF="http://www.fsf.org/copyleft/gpl.html">GNU General Public License</A>.
+
+<P>
+There is absolutely NO WARRANTY for JFlex, its code and its documentation.
+
+<P>
+The code generated by JFlex inherits the copyright of the specification it
+was produced from. If it was your specification, you may use the generated 
+code without restriction.
+
+<P>
+See the file <A NAME="tex2html13"
+  HREF="COPYRIGHT"><TT>COPYRIGHT</TT></A>
+for more information.  
+
+<P>
+
+<H2><A NAME="SECTION000120000000000000000"></A><A NAME="References"></A><BR>
+Bibliography
+</H2><DL COMPACT><DD> 
+
+<P>
+<P></P><DT><A NAME="Aho">1</A>
+<DD>
+  A.&nbsp;Aho, R.&nbsp;Sethi, J.&nbsp;Ullman, <EM>Compilers: Principles, Techniques, and Tools</EM>, 1986
+
+<P>
+<P></P><DT><A NAME="Appel">2</A>
+<DD>
+  A.&nbsp;W.&nbsp;Appel, <EM>Modern Compiler Implementation in Java: basic techniques</EM>, 1997
+
+<P>
+<P></P><DT><A NAME="JLex">3</A>
+<DD>
+ E.&nbsp;Berk, <EM>JLex: A lexical analyser generator for Java</EM>,
+<BR> <A NAME="tex2html14"
+  HREF="http://www.cs.princeton.edu/~appel/modern/java/JLex/"><TT>http://www.cs.princeton.edu/~appel/modern/java/JLex/</TT></A>
+<P>
+<P></P><DT><A NAME="fast">4</A>
+<DD>
+  K.&nbsp;Brouwer, W.&nbsp;Gellerich,E.&nbsp;Ploedereder, 
+  <EM>Myths and Facts about the Efficient Implementation of Finite Automata and Lexical Analysis</EM>, 
+  in: Proceedings of the 7th International Conference on Compiler Construction (CC '98), 1998
+
+<P>
+<P></P><DT><A NAME="unicode_rep">5</A>
+<DD>
+  M.&nbsp;Davis, <EM>Unicode Regular Expression Guidelines</EM>, Unicode Technical Report #18, 2000
+<BR>  <A NAME="tex2html15"
+  HREF="http://www.unicode.org/unicode/reports/tr18/tr18-5.1.html"><TT>http://www.unicode.org/unicode/reports/tr18/tr18-5.1.html</TT></A>
+<P>
+<P></P><DT><A NAME="ParseTable">6</A>
+<DD>
+ P.&nbsp;Dencker, K.&nbsp;D&#252;rre, J.&nbsp;Henft, <EM>Optimization of Parser Tables for portable Compilers</EM>, 
+ in: ACM Transactions on Programming Languages and Systems 6(4), 1984
+
+<P>
+<P></P><DT><A NAME="LangSpec">7</A>
+<DD>
+  J.&nbsp;Gosling, B.&nbsp;Joy, G.&nbsp;Steele, <EM>The Java Language Specifcation</EM>, 1996,
+<BR>  <A NAME="tex2html16"
+  HREF="http://www.javasoft.com/docs/books/jls/"><TT>http://www.javasoft.com/docs/books/jls/</TT></A>
+<P>
+<P></P><DT><A NAME="CUP">8</A>
+<DD>
+  S.&nbsp;E.&nbsp;Hudson, <EM>CUP LALR Parser Generator for Java</EM>,
+<BR>  <A NAME="tex2html17"
+  HREF="http://www.cs.princeton.edu/~appel/modern/java/CUP/"><TT>http://www.cs.princeton.edu/~appel/modern/java/CUP/</TT></A>
+<P>
+<P></P><DT><A NAME="BYaccJ">9</A>
+<DD>
+  B.&nbsp;Jamison, <EM>BYacc/J</EM>,
+<BR>  <A NAME="tex2html18"
+  HREF="http://troi.lincom-asg.com/~rjamison/byacc/"><TT>http://troi.lincom-asg.com/~rjamison/byacc/</TT></A>
+<P>
+<P></P><DT><A NAME="MachineSpec">10</A>
+<DD>
+ T.&nbsp;Lindholm, F.&nbsp;Yellin, <EM>The Java Virtual Machine Specification</EM>, 1996,
+<BR> <A NAME="tex2html19"
+  HREF="http://www.javasoft.com/docs/books/vmspec/"><TT>http://www.javasoft.com/docs/books/vmspec/</TT></A>
+<P>
+<P></P><DT><A NAME="flex">11</A>
+<DD>
+ V.&nbsp;Paxon, <EM>flex - The fast lexical analyzer generator</EM>, 1995
+
+<P>
+<P></P><DT><A NAME="SparseTable">12</A>
+<DD>
+  R.&nbsp;E. Tarjan, A.&nbsp;Yao, <EM>Storing a Sparse Table</EM>, in: Communications of the ACM 22(11), 1979 
+
+<P>
+<P></P><DT><A NAME="Maurer">13</A>
+<DD>
+  R.&nbsp;Wilhelm, D.&nbsp;Maurer, <EM>&#220;bersetzerbau</EM>, Berlin 1997<SUP>2</SUP>
+
+<P>
+</DL>
+
+<P>
+<BR><HR><H4>Footnotes</H4>
+<DL>
+<DT><A NAME="foot32">... Java</A><A NAME="foot32"
+ HREF="manual.html#tex2html2"><SUP><IMG  ALIGN="BOTTOM" BORDER="1" ALT="[*]" SRC="footnote.png"></SUP></A>
+<DD>Java is a trademark of 
+Sun Microsystems, Inc., and refers to Sun's Java programming language. 
+JFlex is not sponsored by or affiliated with Sun Microsystems, Inc.
+
+</DL><BR><HR>
+<ADDRESS>
+Mon Apr 12 20:58:12 EST 2004, <a href="http://www.doclsf.de">Gerwin Klein</a>
+</ADDRESS>
+</BODY>
+</HTML>

diff --git a/tools/lexer/jflex-1.4/doc/manual.pdf b/tools/lexer/jflex-1.4/doc/manual.pdf
new file mode 100644 (file)
index 0000000..96734f8
Binary files /dev/null and b/tools/lexer/jflex-1.4/doc/manual.pdf differ

diff --git a/tools/lexer/jflex-1.4/doc/manual.ps.gz b/tools/lexer/jflex-1.4/doc/manual.ps.gz
new file mode 100644 (file)
index 0000000..50c8ab5
Binary files /dev/null and b/tools/lexer/jflex-1.4/doc/manual.ps.gz differ

diff --git a/tools/lexer/jflex-1.4/examples/binary/Makefile b/tools/lexer/jflex-1.4/examples/binary/Makefile
new file mode 100644 (file)
index 0000000..52607f7
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/binary/Makefile
@@ -0,0 +1,22 @@
+
+JAVA  = java
+JAVAC = javac
+JFLEX = jflex
+
+# -------------------------------------------------
+
+all: run
+
+run: compile
+       $(JAVA) BinaryLexer *
+
+compile: BinaryLexer.class StraightStreamReader.class
+
+clean:
+       rm -f *~ *.class BinaryLexer.java
+
+%.class: %.java
+       $(JAVAC) $^
+
+BinaryLexer.java: binary.flex
+       $(JFLEX) $^

diff --git a/tools/lexer/jflex-1.4/examples/binary/README b/tools/lexer/jflex-1.4/examples/binary/README
new file mode 100644 (file)
index 0000000..8fceab4
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/binary/README
@@ -0,0 +1,18 @@
+This directory contains a small example of a scanner for binary files.
+
+The example tries to detect if an input file is a java class file by
+checking if the first 4 bytes contain the magic number 0xCAFEBABE.
+It uses a custom Reader class that copies bytes to characters one to
+one. The characters the scanner gets to read are the bytes from the
+input stream without any conversion. Many thanks to Stephen Ostermiller
+for providing the custom Reader class, the original version of that
+class is available from http://www.smo.f2s.com/utils/
+
+See the JFlex manual for details on how to scan binaries and how
+this is different from scanning text files.
+
+Files:
+binary.flex                 the scanner spec
+StraightStreamReader.java   the custom reader class (by Stephen Ostermiller)
+Makefile                    rules for make to compile and run the example
+README                      this file

diff --git a/tools/lexer/jflex-1.4/examples/binary/StraightStreamReader.java b/tools/lexer/jflex-1.4/examples/binary/StraightStreamReader.java
new file mode 100644 (file)
index 0000000..7f418b5
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/binary/StraightStreamReader.java
@@ -0,0 +1,287 @@
+/*
+ * An InputStreamReader that does no character encoding translations.
+ * Copyright (C) 2001 Stephen Ostermiller <utils@Ostermiller.com>
+ *
+ * Changes: (2001 by Gerwin Klein <lsf@jflex.de>)
+ *          - commented out package declaration for the example
+ *          - original version at http://www.smo.f2s.com/utils/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * See COPYING.TXT for details.
+ */
+ 
+// package com.Ostermiller.util;
+
+import java.io.*;
+
+/**
+ * A StraightStreamReader is a bridge from byte streams to character streams: It reads bytes
+ * and translates them into characters without using a character encoding.  The characters
+ * that a StraightStreamReader returns may not be valid unicode characters but they are
+ * guaranteed to be in the 0x00 to 0xFF range.
+ * <P>
+ * Most of the time you want to do character encoding translation when translating bytes to
+ * characters.  If you are planning on displaying the text, you should always do this and should
+ * use an InputStreamReader for the purpose.  Sometimes it is useful to treat characters as bytes
+ * with some extra bits.  In these cases you would want to use a StraightStreamReader.
+ * <P>
+ * For top efficiency, consider wrapping an StraightStreamReader within a BufferedReader. For example:<br>
+ * <code>BufferedReader in = new BufferedReader(new StraightStreamReader(System.in));</code>
+ */
+public class StraightStreamReader extends Reader{
+
+    /**
+     * The input stream from which all methods in this class read.
+     */
+    private InputStream in;
+
+    /**
+     * A byte array to be used for calls to the InputStream.  This
+     * is cached as a class variable to avoid object creation and
+     * deletion each time a read is called.  This buffer may be
+     * null and may not be large enough.  Make sure to check it
+     * before using it.
+     */
+    private byte[] buffer;
+
+    /**
+     * Create a StraightStreamReader from an InputStream
+     *
+     * @param in InputStream to wrap a Reader around.
+        */
+       public StraightStreamReader(InputStream in) {
+               this.in = in;
+       }
+
+    /**
+     * Close the stream.
+     *
+     * @throws IOException If an I/O error occurs
+     */
+       public void close() throws IOException {
+               in.close();
+       }
+
+    /**
+     * Mark the present position in the stream. Subsequent calls to reset() 
+        * will attempt to reposition the stream to this point. Not all 
+        * character-input streams support the mark() operation.
+     *
+     * @param readAheadLimit Limit on the number of characters that may be read 
+        *    while still preserving the mark. After reading this many characters, 
+        *    attempting to reset the stream may fail.
+     * @throws IOException If the stream does not support mark(), or if some other I/O error occurs
+     */
+    public void mark(int readAheadLimit) throws IOException {
+       in.mark(readAheadLimit);
+    }
+
+    /**
+     * Tell whether this stream supports the mark() operation.
+     *
+        * @return true if and only if this stream supports the mark operation.
+     */
+       public boolean markSupported(){
+               return in.markSupported();
+       } 
+
+    /**
+        * Read a single character. This method will block until a character is available, an 
+        * I/O error occurs, or the end of the stream is reached.
+     * 
+        * @return The character read, as an integer in the range 0 to 256 (0x00-0xff), or -1 if 
+        *    the end of the stream has been reached
+        * @throws IOException If an I/O error occurs
+     */
+       public int read() throws IOException { 
+        return in.read();
+       }
+
+    /**
+        * Read characters into an array. This method will block until some input is available,
+        * an I/O error occurs, or the end of the stream is reached.
+     *
+        * @param cbuf Destination buffer
+     * @return The number of bytes read, or -1 if the end of the stream has been reached
+        * @throws IOException If an I/O error occurs
+     */
+       public int read(char[] cbuf) throws IOException {
+               return read(cbuf, 0, cbuf.length);
+       }
+    
+    /**
+        * Read characters into an array. This method will block until some input is available,
+        * an I/O error occurs, or the end of the stream is reached.
+     *
+        * @param cbuf Destination buffer
+        * @param off Offset at which to start storing characters
+        * @param len Maximum number of characters to read
+        * @return The number of bytes read, or -1 if the end of the stream has been reached
+        * @throws IOException If an I/O error occurs
+     */
+       public int read(char[] cbuf, int off, int len) throws IOException {
+        // ensure the capacity of the buffer that we will be using
+        // to read from the input stream
+               if (buffer == null || buffer.length < len){
+                       buffer = new byte[len];
+        }
+        // read from the input stream and copy it to the character array
+        int length = in.read(buffer, 0, len);
+        for (int i=0; i<length; i++){
+            cbuf[off+i] = (char)(0xFF & buffer[i]);
+               }
+               return length;
+       }
+
+       /**
+        * Tell whether this stream is ready to be read.
+     *
+        * @return True if the next read() is guaranteed not to block for input, false otherwise.
+     *    Note that returning false does not guarantee that the next read will block.
+        * @throws IOException If an I/O error occurs
+     */
+       public boolean ready() throws IOException {
+               return (in.available() > 0);
+       }
+
+       /**
+        * Reset the stream. If the stream has been marked, then attempt to reposition it at the mark. 
+        * If the stream has not been marked, then attempt to reset it in some way appropriate to the 
+        * particular stream, for example by repositioning it to its starting point. Not all 
+        * character-input streams support the reset() operation, and some support reset() 
+        * without supporting mark().
+     *
+        * @throws IOException If the stream has not been marked, or if the mark has been invalidated, 
+        *    or if the stream does not support reset(), or if some other I/O error occurs
+     */
+       public void reset() throws IOException {
+               in.reset();
+       }
+
+       /**
+        * Skip characters. This method will block until some characters are available, 
+        * an I/O error occurs, or the end of the stream is reached.
+        *
+        * @param n The number of characters to skip
+        * @return The number of characters actually skipped
+        * @throws IllegalArgumentException If n is negative
+     * @throws IOException If an I/O error occurs
+     */
+       public long skip(long n) throws IOException {
+        return in.skip(n);
+       }
+
+    /**
+     * Regression test for this class.  If this class is working, this should
+     * run and print no errors.
+     * <P>
+     * This method creates a tempory file in the working directory called "test.txt".
+     * This file should not exist before hand, and the program should have create,
+     * read, write, and delete access to this file.
+     *
+     * @param args command line arguments (ignored)
+     */
+    private static void main(String[] args){
+        try {
+            File f = new File("test.txt");
+            if (f.exists()){
+                throw new IOException(f + " already exists.  I don't want to overwrite it.");
+            }
+            StraightStreamReader in;
+            char[] cbuf = new char[0x1000];
+            int read;
+            int totRead;
+
+            // write a file with all possible values of bytes
+                       FileOutputStream out = new FileOutputStream(f);
+            for (int i=0x00; i<0x100; i++){
+                out.write(i);
+            }
+            out.close();
+
+            // read it back using the read single character method
+            in = new StraightStreamReader(new FileInputStream(f));
+            for (int i=0x00; i<0x100; i++){
+                read = in.read();
+                if (read != i){
+                       System.err.println("Error: " + i + " read as " + read);
+                }
+            }
+            in.close();
+
+            // read as much of it back as possible with one simple buffer read.
+            in = new StraightStreamReader(new FileInputStream(f));
+            totRead = in.read(cbuf);
+            if (totRead != 0x100){
+                System.err.println("Simple buffered read did not read the full amount: 0x" + Integer.toHexString(totRead));
+            }
+            for (int i=0x00; i<totRead; i++){
+               if (cbuf[i] != i){
+                       System.err.println("Error: 0x" + i + " read as 0x" + cbuf[i]);
+                }
+            }
+            in.close();
+
+            // read it back using buffer read method.
+            in = new StraightStreamReader(new FileInputStream(f));
+            totRead = 0;
+            while (totRead <= 0x100 && (read = in.read(cbuf, totRead, 0x100 - totRead)) > 0){
+                totRead += read;
+                       }
+            if (totRead != 0x100){
+                System.err.println("Not enough read. Bytes read: " + Integer.toHexString(totRead));
+            }
+            for (int i=0x00; i<totRead; i++){
+               if (cbuf[i] != i){
+                       System.err.println("Error: 0x" + i + " read as 0x" + cbuf[i]);
+                }
+            }
+            in.close();
+
+            // read it back using an offset buffer read method.
+            in = new StraightStreamReader(new FileInputStream(f));
+            totRead = 0;
+            while (totRead <= 0x100 && (read = in.read(cbuf, totRead+0x123, 0x100 - totRead)) > 0){
+                totRead += read;
+                       }
+            if (totRead != 0x100){
+                System.err.println("Not enough read. Bytes read: " + Integer.toHexString(totRead));
+            }
+            for (int i=0x00; i<totRead; i++){
+               if (cbuf[i+0x123] != i){
+                       System.err.println("Error: 0x" + i + " read as 0x" + cbuf[i+0x123]);
+                }
+            }
+            in.close();
+
+            // read it back using a partial offset buffer read method.
+            in = new StraightStreamReader(new FileInputStream(f));
+            totRead = 0;
+            while (totRead <= 0x100 && (read = in.read(cbuf, totRead+0x123, 7)) > 0){
+                totRead += read;
+                       }
+            if (totRead != 0x100){
+                System.err.println("Not enough read. Bytes read: " + Integer.toHexString(totRead));
+            }
+            for (int i=0x00; i<totRead; i++){
+               if (cbuf[i+0x123] != i){
+                       System.err.println("Error: 0x" + i + " read as 0x" + cbuf[i+0x123]);
+                }
+            }
+            in.close();
+
+            f.delete();
+        } catch (IOException x){
+            System.err.println(x.getMessage());
+               }
+       }
+}

diff --git a/tools/lexer/jflex-1.4/examples/binary/binary.flex b/tools/lexer/jflex-1.4/examples/binary/binary.flex
new file mode 100644 (file)
index 0000000..6771da6
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/binary/binary.flex
@@ -0,0 +1,52 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 1998-2004  Gerwin Klein <lsf@jflex.de>                    *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+import java.io.*;
+
+%%
+
+%class BinaryLexer
+%8bit
+
+%int
+
+%{
+  public static void main(String [] argv) {
+    for (int i = 0; i < argv.length; i++) {
+      try {
+        System.out.print("["+argv[i]+"] is ");
+        BinaryLexer l = new BinaryLexer(new StraightStreamReader(new FileInputStream(argv[i])));
+        l.yylex();
+      }
+      catch (Exception e) {
+        e.printStackTrace(System.out);
+        System.exit(1);
+      }
+    }
+  }
+%}
+
+magic = \xCA \xFE \xBA \xBE
+
+%%
+
+{magic} [^]+  { System.out.println("a class file"); }
+
+[^]+          { System.out.println("not a class file"); }

diff --git a/tools/lexer/jflex-1.4/examples/byaccj/Makefile b/tools/lexer/jflex-1.4/examples/byaccj/Makefile
new file mode 100644 (file)
index 0000000..28a361d
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/byaccj/Makefile
@@ -0,0 +1,27 @@
+# only works with the Java extension of yacc: 
+# byacc/j from http://troi.lincom-asg.com/~rjamison/byacc/
+
+JFLEX  = jflex 
+BYACCJ = yacc -J
+JAVAC  = javac
+
+# targets:
+
+all: Parser.class
+
+run: Parser.class
+       java Parser
+
+build: clean Parser.class
+
+clean:
+       rm -f *~ *.class *.java
+
+Parser.class: Yylex.java Parser.java
+       $(JAVAC) Parser.java
+
+Yylex.java: calc.flex
+       $(JFLEX) calc.flex
+
+Parser.java: calc.y
+       $(BYACCJ) calc.y

diff --git a/tools/lexer/jflex-1.4/examples/byaccj/README b/tools/lexer/jflex-1.4/examples/byaccj/README
new file mode 100644 (file)
index 0000000..ab6831e
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/byaccj/README
@@ -0,0 +1,22 @@
+This directory contains an interoperability example for BYacc/J and JFlex
+(a small calculator)
+
+You need the parser generator byacc/j from
+http://troi.lincom-asg.com/~rjamison/byacc/
+for this example to work (version byacc/j 1.1 has been tested).
+
+Use the Makefile to generate the lexer and parser.
+The example can then be started with
+
+java Parser
+
+
+files:
+
+calc.flex
+  JFlex specification for the lexical part of the arithmetic expressions
+
+calc.y
+  BYacc/J specification and main program for the calculator
+
+

diff --git a/tools/lexer/jflex-1.4/examples/byaccj/calc.flex b/tools/lexer/jflex-1.4/examples/byaccj/calc.flex
new file mode 100644 (file)
index 0000000..8ad62bc
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/byaccj/calc.flex
@@ -0,0 +1,62 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2000 Gerwin Klein <lsf@jflex.de>                          *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * Thanks to Larry Bell and Bob Jamison for suggestions and comments.      *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+%%
+
+%byaccj
+
+%{
+  private Parser yyparser;
+
+  public Yylex(java.io.Reader r, Parser yyparser) {
+    this(r);
+    this.yyparser = yyparser;
+  }
+%}
+
+NUM = [0-9]+ ("." [0-9]+)?
+NL  = \n | \r | \r\n
+
+%%
+
+/* operators */
+"+" | 
+"-" | 
+"*" | 
+"/" | 
+"^" | 
+"(" | 
+")"    { return (int) yycharat(0); }
+
+/* newline */
+{NL}   { return Parser.NL; }
+
+/* float */
+{NUM}  { yyparser.yylval = new ParserVal(Double.parseDouble(yytext()));
+         return Parser.NUM; }
+
+/* whitespace */
+[ \t]+ { }
+
+\b     { System.err.println("Sorry, backspace doesn't work"); }
+
+/* error fallback */
+[^]    { System.err.println("Error: unexpected character '"+yytext()+"'"); return -1; }

diff --git a/tools/lexer/jflex-1.4/examples/byaccj/calc.y b/tools/lexer/jflex-1.4/examples/byaccj/calc.y
new file mode 100644 (file)
index 0000000..7ab42c2
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/byaccj/calc.y
@@ -0,0 +1,112 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001 Gerwin Klein <lsf@jflex.de>                          *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This is a modified version of the example from                          *
+ *   http://www.lincom-asg.com/~rjamison/byacc/                            *
+ *                                                                         *
+ * Thanks to Larry Bell and Bob Jamison for suggestions and comments.      *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+%{
+  import java.io.*;
+%}
+      
+%token NL          /* newline  */
+%token <dval> NUM  /* a number */
+
+%type <dval> exp
+
+%left '-' '+'
+%left '*' '/'
+%left NEG          /* negation--unary minus */
+%right '^'         /* exponentiation        */
+      
+%%
+
+input:   /* empty string */
+       | input line
+       ;
+      
+line:    NL      { if (interactive) System.out.print("Expression: "); }
+       | exp NL  { System.out.println(" = " + $1); 
+                   if (interactive) System.out.print("Expression: "); }
+       ;
+      
+exp:     NUM                { $$ = $1; }
+       | exp '+' exp        { $$ = $1 + $3; }
+       | exp '-' exp        { $$ = $1 - $3; }
+       | exp '*' exp        { $$ = $1 * $3; }
+       | exp '/' exp        { $$ = $1 / $3; }
+       | '-' exp  %prec NEG { $$ = -$2; }
+       | exp '^' exp        { $$ = Math.pow($1, $3); }
+       | '(' exp ')'        { $$ = $2; }
+       ;
+
+%%
+
+  private Yylex lexer;
+
+
+  private int yylex () {
+    int yyl_return = -1;
+    try {
+      yylval = new ParserVal(0);
+      yyl_return = lexer.yylex();
+    }
+    catch (IOException e) {
+      System.err.println("IO error :"+e);
+    }
+    return yyl_return;
+  }
+
+
+  public void yyerror (String error) {
+    System.err.println ("Error: " + error);
+  }
+
+
+  public Parser(Reader r) {
+    lexer = new Yylex(r, this);
+  }
+
+
+  static boolean interactive;
+
+  public static void main(String args[]) throws IOException {
+    System.out.println("BYACC/Java with JFlex Calculator Demo");
+
+    Parser yyparser;
+    if ( args.length > 0 ) {
+      // parse a file
+      yyparser = new Parser(new FileReader(args[0]));
+    }
+    else {
+      // interactive mode
+      System.out.println("[Quit with CTRL-D]");
+      System.out.print("Expression: ");
+      interactive = true;
+           yyparser = new Parser(new InputStreamReader(System.in));
+    }
+
+    yyparser.yyparse();
+    
+    if (interactive) {
+      System.out.println();
+      System.out.println("Have a nice day");
+    }
+  }

diff --git a/tools/lexer/jflex-1.4/examples/cup/Main.java b/tools/lexer/jflex-1.4/examples/cup/Main.java
new file mode 100644 (file)
index 0000000..fba4532
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/cup/Main.java
@@ -0,0 +1,39 @@
+/*
+  This example comes from a short article series in the Linux 
+  Gazette by Richard A. Sevenich and Christopher Lopes, titled
+  "Compiler Construction Tools". The article series starts at
+
+  http://www.linuxgazette.com/issue39/sevenich.html
+
+  Small changes and updates to newest JFlex+Cup versions 
+  by Gerwin Klein
+*/
+
+/*
+  Commented By: Christopher Lopes
+  File Name: Main.java
+  To Create: 
+  After the scanner, lcalc.flex, and the parser, ycalc.cup, have been created.
+  > javac Main.java
+  
+  To Run: 
+  > java Main test.txt
+  where test.txt is an test input file for the calculator.
+*/
+
+import java.io.FileReader;
+   
+public class Main {
+  static public void main(String argv[]) {    
+    /* Start the parser */
+    try {
+      parser p = new parser(new Lexer(new FileReader(argv[0])));
+      Object result = p.parse().value;      
+    } catch (Exception e) {
+      /* do cleanup here -- possibly rethrow e */
+      e.printStackTrace();
+    }
+  }
+}
+
+

diff --git a/tools/lexer/jflex-1.4/examples/cup/Makefile b/tools/lexer/jflex-1.4/examples/cup/Makefile
new file mode 100644 (file)
index 0000000..fff6fbc
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/cup/Makefile
@@ -0,0 +1,26 @@
+JAVA=java
+JAVAC=javac
+JFLEX=jflex
+CUP=$(JAVA) java_cup.Main <
+
+all: test
+
+test: output.txt
+       @(diff output.txt output.good && echo "Test OK!") || echo "Test failed!"
+
+output.txt: Main.class test.txt
+       $(JAVA) Main test.txt > output.txt
+
+Main.class: Main.java Lexer.java parser.java
+
+%.class: %.java
+       $(JAVAC) $^
+
+Lexer.java: lcalc.flex
+       $(JFLEX) lcalc.flex
+
+parser.java: ycalc.cup
+       $(CUP) ycalc.cup
+
+clean:
+       rm -f parser.java Lexer.java sym.java output.txt *.class *~

diff --git a/tools/lexer/jflex-1.4/examples/cup/README b/tools/lexer/jflex-1.4/examples/cup/README
new file mode 100644 (file)
index 0000000..6b0d333
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/cup/README
@@ -0,0 +1,30 @@
+A small JFlex+Cup example
+
+It comes from a short article series in the Linux Gazette by Richard
+A. Sevenich and Christopher Lopes, titled "Compiler Construction
+Tools". The article series starts at
+
+http://www.linuxgazette.com/issue39/sevenich.html
+
+Small changes and updates to newest JFlex+Cup versions by Gerwin Klein
+
+
+To compile:
+>jflex lcalc.flex
+>java java_cup.Main < ycalc.cup
+>javac Main.java
+
+To run:
+>java Main test.txt
+
+
+Files:
+
+Main.java       demo of a main program
+Makefile        makefile to compile and test the example
+README          this file
+lcalc.flex      the lexer spec
+output.good     how the output should look like for the test
+ycalc.cup       the parser spec
+test.txt        sample input for testing
+

diff --git a/tools/lexer/jflex-1.4/examples/cup/lcalc.flex b/tools/lexer/jflex-1.4/examples/cup/lcalc.flex
new file mode 100644 (file)
index 0000000..181bf01
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/cup/lcalc.flex
@@ -0,0 +1,142 @@
+/*
+  This example comes from a short article series in the Linux 
+  Gazette by Richard A. Sevenich and Christopher Lopes, titled
+  "Compiler Construction Tools". The article series starts at
+
+  http://www.linuxgazette.com/issue39/sevenich.html
+
+  Small changes and updates to newest JFlex+Cup versions 
+  by Gerwin Klein
+*/
+
+/*
+  Commented By: Christopher Lopes
+  File Name: lcalc.flex
+  To Create: > jflex lcalc.flex
+
+  and then after the parser is created
+  > javac Lexer.java
+*/
+   
+/* --------------------------Usercode Section------------------------ */
+   
+import java_cup.runtime.*;
+      
+%%
+   
+/* -----------------Options and Declarations Section----------------- */
+   
+/* 
+   The name of the class JFlex will create will be Lexer.
+   Will write the code to the file Lexer.java. 
+*/
+%class Lexer
+
+/*
+  The current line number can be accessed with the variable yyline
+  and the current column number with the variable yycolumn.
+*/
+%line
+%column
+    
+/* 
+   Will switch to a CUP compatibility mode to interface with a CUP
+   generated parser.
+*/
+%cup
+   
+/*
+  Declarations
+   
+  Code between %{ and %}, both of which must be at the beginning of a
+  line, will be copied letter to letter into the lexer class source.
+  Here you declare member variables and functions that are used inside
+  scanner actions.  
+*/
+%{   
+    /* To create a new java_cup.runtime.Symbol with information about
+       the current token, the token will have no value in this
+       case. */
+    private Symbol symbol(int type) {
+        return new Symbol(type, yyline, yycolumn);
+    }
+    
+    /* Also creates a new java_cup.runtime.Symbol with information
+       about the current token, but this object has a value. */
+    private Symbol symbol(int type, Object value) {
+        return new Symbol(type, yyline, yycolumn, value);
+    }
+%}
+   
+
+/*
+  Macro Declarations
+  
+  These declarations are regular expressions that will be used latter
+  in the Lexical Rules Section.  
+*/
+   
+/* A line terminator is a \r (carriage return), \n (line feed), or
+   \r\n. */
+LineTerminator = \r|\n|\r\n
+   
+/* White space is a line terminator, space, tab, or line feed. */
+WhiteSpace     = {LineTerminator} | [ \t\f]
+   
+/* A literal integer is is a number beginning with a number between
+   one and nine followed by zero or more numbers between zero and nine
+   or just a zero.  */
+dec_int_lit = 0 | [1-9][0-9]*
+   
+/* A identifier integer is a word beginning a letter between A and
+   Z, a and z, or an underscore followed by zero or more letters
+   between A and Z, a and z, zero and nine, or an underscore. */
+dec_int_id = [A-Za-z_][A-Za-z_0-9]*
+   
+%%
+/* ------------------------Lexical Rules Section---------------------- */
+   
+/*
+   This section contains regular expressions and actions, i.e. Java
+   code, that will be executed when the scanner matches the associated
+   regular expression. */
+   
+   /* YYINITIAL is the state at which the lexer begins scanning.  So
+   these regular expressions will only be matched if the scanner is in
+   the start state YYINITIAL. */
+   
+<YYINITIAL> {
+   
+    /* Return the token SEMI declared in the class sym that was found. */
+    ";"                { return symbol(sym.SEMI); }
+   
+    /* Print the token found that was declared in the class sym and then
+       return it. */
+    "+"                { System.out.print(" + "); return symbol(sym.PLUS); }
+    "-"                { System.out.print(" - "); return symbol(sym.MINUS); }
+    "*"                { System.out.print(" * "); return symbol(sym.TIMES); }
+    "/"                { System.out.print(" / "); return symbol(sym.DIVIDE); }
+    "("                { System.out.print(" ( "); return symbol(sym.LPAREN); }
+    ")"                { System.out.print(" ) "); return symbol(sym.RPAREN); }
+   
+    /* If an integer is found print it out, return the token NUMBER
+       that represents an integer and the value of the integer that is
+       held in the string yytext which will get turned into an integer
+       before returning */
+    {dec_int_lit}      { System.out.print(yytext());
+                         return symbol(sym.NUMBER, new Integer(yytext())); }
+   
+    /* If an identifier is found print it out, return the token ID
+       that represents an identifier and the default value one that is
+       given to all identifiers. */
+    {dec_int_id}       { System.out.print(yytext());
+                         return symbol(sym.ID, new Integer(1));}
+   
+    /* Don't do anything if whitespace is found */
+    {WhiteSpace}       { /* just skip what was found, do nothing */ }   
+}
+
+
+/* No token was found for the input so through an error.  Print out an
+   Illegal character message with the illegal character that was found. */
+[^]                    { throw new Error("Illegal character <"+yytext()+">"); }

diff --git a/tools/lexer/jflex-1.4/examples/cup/output.good b/tools/lexer/jflex-1.4/examples/cup/output.good
new file mode 100644 (file)
index 0000000..cbceb63
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/cup/output.good
@@ -0,0 +1,4 @@
+2 + 4 = 6
+5 *  ( 6 - 3 )  + 1 = 16
+6 / 3 * 5 + 20 = 30
+4 * 76 / 31 = 9

diff --git a/tools/lexer/jflex-1.4/examples/cup/test.txt b/tools/lexer/jflex-1.4/examples/cup/test.txt
new file mode 100644 (file)
index 0000000..3f8344b
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/cup/test.txt
@@ -0,0 +1,4 @@
+2+4; 
+5*(6-3)+1; 
+6/3*5+20; 
+4*76/31;

diff --git a/tools/lexer/jflex-1.4/examples/cup/ycalc.cup b/tools/lexer/jflex-1.4/examples/cup/ycalc.cup
new file mode 100644 (file)
index 0000000..a67a20d
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/cup/ycalc.cup
@@ -0,0 +1,234 @@
+/*
+  This example comes from a short article series in the Linux 
+  Gazette by Richard A. Sevenich and Christopher Lopes, titled
+  "Compiler Construction Tools". The article series starts at
+
+  http://www.linuxgazette.com/issue39/sevenich.html
+
+  Small changes and updates to newest JFlex+Cup versions 
+  by Gerwin Klein
+*/
+
+/*
+  Commented By: Christopher Lopes
+  File Name: ycalc.cup
+  To Create: > java java_cup.Main < ycalc.cup
+*/
+   
+   
+/* ----------------------Preliminary Declarations Section--------------------*/
+   
+/* Import the class java_cup.runtime.*  */
+import java_cup.runtime.*;
+   
+/* Parser code to change the way the parser reports errors (include
+   line and column number of the error). */
+parser code {:
+    
+    /* Change the method report_error so it will display the line and
+       column of where the error occurred in the input as well as the
+       reason for the error which is passed into the method in the
+       String 'message'. */
+    public void report_error(String message, Object info) {
+   
+        /* Create a StringBuffer called 'm' with the string 'Error' in it. */
+        StringBuffer m = new StringBuffer("Error");
+   
+        /* Check if the information passed to the method is the same
+           type as the type java_cup.runtime.Symbol. */
+        if (info instanceof java_cup.runtime.Symbol) {
+            /* Declare a java_cup.runtime.Symbol object 's' with the
+               information in the object info that is being typecasted
+               as a java_cup.runtime.Symbol object. */
+            java_cup.runtime.Symbol s = ((java_cup.runtime.Symbol) info);
+   
+            /* Check if the line number in the input is greater or
+               equal to zero. */
+            if (s.left >= 0) {                
+                /* Add to the end of the StringBuffer error message
+                   the line number of the error in the input. */
+                m.append(" in line "+(s.left+1));   
+                /* Check if the column number in the input is greater
+                   or equal to zero. */
+                if (s.right >= 0)                    
+                    /* Add to the end of the StringBuffer error message
+                       the column number of the error in the input. */
+                    m.append(", column "+(s.right+1));
+            }
+        }
+   
+        /* Add to the end of the StringBuffer error message created in
+           this method the message that was passed into this method. */
+        m.append(" : "+message);
+   
+        /* Print the contents of the StringBuffer 'm', which contains
+           an error message, out on a line. */
+        System.err.println(m);
+    }
+   
+    /* Change the method report_fatal_error so when it reports a fatal
+       error it will display the line and column number of where the
+       fatal error occurred in the input as well as the reason for the
+       fatal error which is passed into the method in the object
+       'message' and then exit.*/
+    public void report_fatal_error(String message, Object info) {
+        report_error(message, info);
+        System.exit(1);
+    }
+:};
+   
+
+   
+/* ------------Declaration of Terminals and Non Terminals Section----------- */
+   
+/* Terminals (tokens returned by the scanner).  
+
+   Terminals that have no value are listed first and then terminals
+   that do have an value, in this case an integer value, are listed on
+   the next line down. */
+terminal           SEMI, PLUS, MINUS, TIMES, DIVIDE, LPAREN, RPAREN;
+terminal Integer   NUMBER, ID;
+   
+/* Non terminals used in the grammar section.  
+
+   Non terminals that have an object value are listed first and then
+   non terminals that have an integer value are listed.  An object
+   value means that it can be any type, it isn't set to a specific
+   type.  So it could be an Integer or a String or whatever. */
+non terminal Object     expr_list, expr_part;
+non terminal Integer    expr, factor, term;
+   
+
+/* -------------Precedence and Associatively of Terminals Section----------- */
+   
+/*
+  Precedence of non terminals could be defined here.  If you do define
+  precedence here you won't need to worry about precedence in the
+  Grammar Section, i.e. that TIMES should have a higher precedence
+  than PLUS.
+  
+  The precedence defined here would look something like this where the
+  lower line always will have higher precedence than the line before it.
+   
+  precedence left PLUS, MINUS; 
+  precedence left TIMES, DIVIDE; 
+*/
+
+
+/* ----------------------------Grammar Section-------------------- */
+   
+/* The grammar for our parser.
+   
+   expr_list ::=   expr_list expr_part
+                 | expr_part
+   expr_part ::=   expr SEMI
+   expr      ::=   factor PLUS expr
+                 | factor MINUS expr
+                 | factor
+   factor    ::=   factor TIMES term
+                 | factor DIVIDE term
+                 | term
+   term     ::=    LPAREN expr RPAREN
+                 | NUMBER
+                 | ID     
+*/
+   
+/* 'expr_list' is the start of our grammar.  It can lead to another
+   'expr_list' followed by an 'expr_part' or it can just lead to an
+   'expr_part'.  The lhs of the non terminals 'expr_list' and
+   'expr_part' that are in the rhs side of the production below need
+   to be found.  Then the rhs sides of those non terminals need to be
+   followed in a similar manner, i.e. if there are any non terminals
+   in the rhs of those productions then the productions with those non
+   terminals need to be found and those rhs's followed.  This process
+   keeps continuing until only terminals are found in the rhs of a
+   production.  Then we can work our way back up the grammar bringing
+   any values that might have been assigned from a terminal. */
+   
+   expr_list ::= expr_list expr_part
+                 |
+                 expr_part;
+   
+/* 'expr_part' is an 'expr' followed by the terminal 'SEMI'.  The ':e'
+   after the non terminal 'expr' is a label an is used to access the
+   value of 'expr' which will be an integer.  The action for the
+   production lies between {: and :}.  This action will print out the
+   line " = + e" where e is the value of 'expr'.  Before the action
+   takes places we need to go deeper into the grammar since 'expr' is
+   a non terminal.  Whenever a non terminal is encountered on the rhs
+   of a production we need to find the rhs of that non terminal until
+   there are no more non terminals in the rhs.  So when we finish
+   going through the grammar and don't encounter any more non
+   terminals in the rhs productions will return until we get back to
+   'expr' and at that point 'expr' will contain an integer value. */
+   
+   expr_part ::= expr:e
+                 {: System.out.println(" = " + e); :}
+                 SEMI
+                 ;
+   
+/* 'expr' can lead to 'factor PLUS expr', 'factor MINUS expr', or
+   'factor'.  The 'TIMES' and 'DIVIDE' productions are not at this
+   level.  They are at a lower level in the grammar which in affect
+   makes them have higher precedence.  Actions for the rhs of the non
+   terminal 'expr' return a value to 'expr'.  This value that is
+   created is an integer and gets stored in 'RESULT' in the action.
+   RESULT is the label that is assigned automatically to the rhs, in
+   this case 'expr'.  If the rhs is just 'factor' then 'f' refers to
+   the non terminal 'factor'.  The value of 'f' is retrieved with the
+   function 'intValue()' and will be stored in 'RESULT'.  In the other
+   two cases 'f' and 'e' refers to the non terminals 'factor' and
+   'expr' respectively with a terminal between them, either 'PLUS' or
+   'MINUS'.  The value of each is retrieved with the same function
+   'intValue'.  The values will be added or subtracted and then the
+   new integer will be stored in 'RESULT'.*/
+   
+   expr      ::= factor:f PLUS expr:e
+                 {: RESULT = new Integer(f.intValue() + e.intValue()); :}
+                 |
+                 factor:f MINUS expr:e
+                 {: RESULT = new Integer(f.intValue() - e.intValue()); :}
+                 |
+                 factor:f
+                 {: RESULT = new Integer(f.intValue()); :}
+                 ;
+   
+/* 'factor' can lead to 'factor TIMES term', 'factor DIVIDE term', or
+   'term'.  Since the productions for TIMES and DIVIDE are lower in
+   the grammar than 'PLUS' and 'MINUS' they will have higher
+   precedence.  The same sort of actions take place in the rhs of
+   'factor' as in 'expr'.  The only difference is the operations that
+   takes place on the values retrieved with 'intValue()', 'TIMES' and
+   'DIVIDE' here instead of 'PLUS' and 'MINUS'.  */
+   
+   factor    ::= factor:f TIMES term:t
+                 {: RESULT = new Integer(f.intValue() * t.intValue()); :}
+                 |
+                 factor:f DIVIDE term:t
+                 {: RESULT = new Integer(f.intValue() / t.intValue()); :}
+                 |
+                 term:t
+                 {: RESULT = new Integer(t.intValue()); :}
+                 ;
+   
+/* 'term' can lead to 'LPAREN expr RPAREN', 'NUMBER', or 'ID'.  The
+   first production has the non terminal 'expr' in it so the
+   production with its lhs side needs to be found and followed.  The
+   next rhs has no non terminals.  So the grammar ends here and can go
+   back up.  When it goes back up it will bring the value that was
+   retrieved when the scanner encounter the token 'NUMBER'.  'RESULT'
+   is assigned 'n', which refers to 'NUMBER', as the action for this
+   production.  The same action occurs for 'ID', except the 'i' is
+   used to refer to 'ID'.  'ID' is also the only thing on the rhs of
+   the production.  And since 'ID' is a terminal the grammar will end
+   here and go back up. */
+   
+   term      ::= LPAREN expr:e RPAREN
+                 {: RESULT = e; :}
+                 |
+                 NUMBER:n
+                 {: RESULT = n; :}
+                 |
+                 ID:i
+                 {: RESULT = i; :}
+                 ;

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/AST.java b/tools/lexer/jflex-1.4/examples/interpreter/AST.java
new file mode 100644 (file)
index 0000000..194a42c
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/AST.java
@@ -0,0 +1,30 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/**
+ * General interface for nodes in the abstract syntax tree. Contains   
+ * only the method toString which is already inherited from Object,   
+ * so the interface doesn't add any functionality. It only provides   
+ * a common super type for all elements in the AST.                   
+ */
+interface AST {
+  public String toString();   // already inherited from Object
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Main.java b/tools/lexer/jflex-1.4/examples/interpreter/Main.java
new file mode 100644 (file)
index 0000000..4fbaf1a
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Main.java
@@ -0,0 +1,88 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.io.Reader;
+
+/**
+ * Main program of the interpreter for the AS programming language.
+ * Based on JFlex/CUP.
+ * 
+ * Steps:
+ * - scanning                               (Yylex)
+ * - context free parsing and AST building  (yyparse)
+ * - build up symbol table                  (setSymtabs)
+ * - check context conditions               (checkcontext)
+ * - prepare interpretation                 (prepInterp)
+ * - start interpretation                   (interpret)
+ */ 
+public class Main {
+
+  public static void main(String [] args) throws Exception {
+    Reader reader = null;
+    
+    if (args.length == 1) {
+      File input = new File(args[0]);
+      if (!input.canRead()) {
+        System.out.println("Error: could not read ["+input+"]");
+      }
+      reader = new FileReader(input);
+    }
+    else {  
+     reader = new InputStreamReader(System.in);
+    }
+
+    Yylex scanner = new Yylex(reader);   // create scanner
+    SymTab symtab = new SymTab();        // set global symbol table    
+    scanner.setSymtab(symtab);
+
+    parser parser = new parser(scanner); // create parser
+    Tprogram syntaxbaum = null;
+
+    try { 
+      syntaxbaum = (Tprogram) parser.parse().value;  // parse
+    }    
+    catch (Exception e) { 
+      e.printStackTrace(); 
+    }
+
+    // System.out.println(symtab);
+    System.out.println(syntaxbaum);
+
+    syntaxbaum.setSymtabs();          // set symbol table
+    // syntaxbaum.printSymtabs();       // print symbol table
+
+    syntaxbaum.checkcontext();        // CoCo (DefVar, DefFun, Arity)
+    if(contexterror>0) return;
+
+    syntaxbaum.prepInterp();          // var. indices and function pointers
+    // im Syntaxbaum setzen
+    syntaxbaum.interpret();           // interpretation
+  }
+
+  static int contexterror = 0;        // number of errors in context conditions
+
+  public static void error(String s) { 
+    System.out.println((contexterror++)+". "+s); 
+  }
+}

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Makefile b/tools/lexer/jflex-1.4/examples/interpreter/Makefile
new file mode 100644 (file)
index 0000000..de5afbe
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Makefile
@@ -0,0 +1,42 @@
+# uses JFlex >= 1.3.2, and CUP >= 0.10j
+#
+# targets:
+#
+# make all                                                            
+#    generates lexer, and parser, and compiles all *.java files
+#                                                                     
+# make run (or just: make)                                           
+#    starts the program on a test example
+#                                                                     
+
+
+
+.SUFFIXES: $(SUFFIXES) .class .java
+
+.java.class:
+       javac $*.java
+
+FILE=  Yylex.java      parser.java    sym.java \
+       SymTab.java     AST.java \
+       Tnumber.java    Tident.java     Texp.java       \
+       Tfun.java       Texpinfix.java  Tuminus.java    \
+       Tboolexp.java   Tifthenelse.java                \
+       Tdekl.java      Tdekllist.java                  \
+       Tparlist.java   Texplist.java   Tprogram.java   \
+       Main.java \
+       SymtabEntry.java STEfun.java    STEvar.java
+
+
+run: all
+       java Main < example.as
+
+all: Yylex.java parser.java $(FILE:java=class)
+
+clean:
+       rm -f *.class *~ *.bak Yylex.java parser.java sym.java
+
+Yylex.java: scanner.flex
+       jflex scanner.flex
+
+parser.java: parser.cup
+       java java_cup.Main -interface < parser.cup

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/README b/tools/lexer/jflex-1.4/examples/interpreter/README
new file mode 100644 (file)
index 0000000..731bd5c
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/README
@@ -0,0 +1,31 @@
+This directory contains an interpreter for a small functional
+programming language (called "AS"). It uses JFlex and CUP.
+
+Directory contents:
+
+- Main.java is the main program. It reads an AS program from
+  std in and interprets it.
+
+- Makefile runs jflex, cup, javac and the compiled interpreter
+
+- Classes beginning with "T" implement the abstract syntax tree.
+  They also contain context condition checking and the interpreter.
+
+- Symtab.java, SymtabEntry.java, STEfun.java, STEvar.java implement
+  the symbol table.
+
+- scanner.lex and parser.cup contain the scanner and parser.
+
+- example.as is an example program in AS
+
+
+The language is described in:
+Manfred Broy: Einfuehrung in die Informatik, Springer Verlag 
+
+The files are a solution to excercise 3.40 in the book:
+Manfred Broy,Bernhard Rumpe: 
+Uebungen zur Einfuehrung in die Informatik --                      
+Strukturierte Aufgabensammlung mit Musterloesungen, 
+Springer Verlag, 2001
+
+Both books are only available in German. Sorry.

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/STEfun.java b/tools/lexer/jflex-1.4/examples/interpreter/STEfun.java
new file mode 100644 (file)
index 0000000..655813b
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/STEfun.java
@@ -0,0 +1,54 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * Symbol table entry for functions.
+ * 
+ * Contains arity and reference to location of definition
+ */ 
+class STEfun extends SymtabEntry {
+  int arity;
+  Tdekl dekl; // location of definition
+  
+  public STEfun(String f, Tdekl d, int a) { 
+    super(f);
+    dekl=d;
+    arity=a;
+  }
+  
+  public int kind() { 
+    return SymtabEntry.FUN; 
+  }
+
+  public String toString() { 
+    return "function    "+name+", arity "+arity; 
+  }
+
+  public int arity() { 
+    return arity; 
+  }
+
+  public Tdekl getDekl() { 
+    return dekl; 
+  }
+}
+
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/STEvar.java b/tools/lexer/jflex-1.4/examples/interpreter/STEvar.java
new file mode 100644 (file)
index 0000000..c4d420d
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/STEvar.java
@@ -0,0 +1,57 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * Symbol table entry for variables.
+ * 
+ * Contains index in the parameter list and a flag if it
+ * is an input variable.
+ */ 
+class STEvar extends SymtabEntry {
+  boolean is_input;           
+  int index;                  
+
+  public STEvar(String v, boolean ii, int ind) {
+    super(v);
+    is_input=ii;
+    index=ind;
+  }
+
+  public int kind() {
+    return SymtabEntry.VAR; 
+  }
+
+  public String toString() {
+    if (is_input) 
+      return "input var "+name+"  ("+index+")";
+    else 
+      return "parameter "+name+"  ("+index+")";
+  }
+
+  public int getIndex() {
+    return index; 
+  }
+
+  public boolean isInput() {
+    return is_input; 
+  }
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/SymTab.java b/tools/lexer/jflex-1.4/examples/interpreter/SymTab.java
new file mode 100644 (file)
index 0000000..676a736
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/SymTab.java
@@ -0,0 +1,79 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+import java.util.Enumeration;
+import java.util.Hashtable;
+
+/**
+ * Symbol table for the interpreter, contains information about
+ * variables and functions.
+ * 
+ * For every binding location of a name a symbol will be created.
+ * The symbol tables are connected hierarchically by pointers to
+ * the predecessor. Lookup takes predecessors into account.
+ */ 
+public class SymTab {
+  Hashtable h;        // contains the liste of words
+                      // key: String, value: SymtabEntry
+  
+  SymTab pred;  // predecessor symbol table (if exists)
+
+  public SymTab() {
+    this(null); 
+  }
+
+  public SymTab(SymTab p) {
+    h    = new Hashtable();
+    pred = p;
+  }
+
+  public boolean enter(String s, SymtabEntry e) {
+    Object value = lookup(s);
+    h.put(s, e);
+    return(value==null);
+  }
+
+  public SymtabEntry lookup(String s) {
+    Object value = h.get(s);
+    if (value==null && pred!=null)
+      value = pred.lookup(s);
+    return ((SymtabEntry)value);
+  }
+
+  public String toString() {    // for output with print
+    String res    = "symbol table\n=============\n";
+    Enumeration e = h.keys();
+    String key;
+    
+    while(e.hasMoreElements()) {
+      key = (String)e.nextElement();
+      res += key+"   \t"+h.get(key)+"\n";
+    }
+
+    if (pred!=null) res+="++ predecessor!\n";
+    return(res);
+  }
+
+  public int size() {
+    return(h.size()); 
+  }
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/SymtabEntry.java b/tools/lexer/jflex-1.4/examples/interpreter/SymtabEntry.java
new file mode 100644 (file)
index 0000000..9eb7f13
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/SymtabEntry.java
@@ -0,0 +1,48 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * Symbol table entry for names, there are subclasses for
+ * variables and functions.
+ * 
+ * Defines constants UNKNOWN, VAR und FUN as kinds of
+ * symbol table entries.
+ */ 
+class SymtabEntry {
+  String name;
+
+  public SymtabEntry(String v) {
+    name=v; 
+  }
+
+  public int kind() {
+    return UNKNOWN; 
+  }
+
+  public String toString() {
+    return("unknown "+name); 
+  }
+
+  static final int UNKNOWN = 12;
+  static final int VAR = 13;
+  static final int FUN = 14;
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Tboolexp.java b/tools/lexer/jflex-1.4/examples/interpreter/Tboolexp.java
new file mode 100644 (file)
index 0000000..59ab0e3
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Tboolexp.java
@@ -0,0 +1,64 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for bool expressions
+ */ 
+class Tboolexp implements AST {
+  Texp exp1, exp2;            // left and right subexpression 
+  char kind;                  // '=', '<' and '!' for "<="
+
+  public Tboolexp(Texp e1, char k, Texp e2) {
+    exp1=e1;
+    kind=k;
+    exp2=e2;
+  }
+
+  public String toString() {
+    if (kind!='!') 
+      return(""+exp1+kind+exp2);
+    else 
+      return(exp1+"<="+exp2);
+  }
+
+  public void checkcontext(SymTab st) { // context conditions 
+    exp1.checkcontext(st);
+    exp2.checkcontext(st);
+  }
+
+  public void prepInterp(SymTab st) {   // set pointers and indices
+    exp1.prepInterp(st);
+    exp2.prepInterp(st);
+  }
+
+  public boolean interpret(int[] in, int[] par) {
+    int e1 = exp1.interpret(in,par);
+    int e2 = exp2.interpret(in,par);
+    switch(kind) {
+    case '=': return(e1==e2);
+    case '<': return(e1<e2);
+    case '!': return(e1<=e2);
+    }
+
+    return(false);     // error
+  }
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Tdekl.java b/tools/lexer/jflex-1.4/examples/interpreter/Tdekl.java
new file mode 100644 (file)
index 0000000..3ac2e6e
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Tdekl.java
@@ -0,0 +1,75 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for function declarations.
+ * 
+ * Also contains a reference to the symbol table of 
+ * the paramaters and their arity.
+ */ 
+class Tdekl implements AST {
+  Tident ident;               // identifier
+  Tparlist parlist;           // liste of parameter
+  Texp exp;                   // function body
+
+  public Tdekl(Tident i, Tparlist p, Texp e) {
+    parlist=p;
+    ident=i;
+    exp=e;
+  }
+
+  public String toString() {
+    return(ident+"("+parlist+") = \n  "+exp); 
+  }
+
+  SymTab params;              // symbol table of the parameters 
+  int arity;                  
+
+  public void setSymtab(SymTab st) {
+    params = new SymTab(st);
+    parlist.setSymtab(params,false,0);
+    arity = params.size();
+    
+    boolean isNew = st.enter(ident.toString(),
+                             new STEfun(ident.toString(),this,arity));
+    // CoCo (Fun)
+    if(!isNew) Main.error("funktion "+ident+" defined twice!");
+  }
+
+  public void printSymtabs() {
+    System.out.print("funktion "+ident.toString()+"\n"+params); 
+  }
+
+  public void checkcontext() {        
+    exp.checkcontext(params);         // CoCo (DefFun,DefVar,Arity)
+  }
+    
+  public void prepInterp(SymTab st) {   // set pointers and indices
+    exp.prepInterp(params);
+  }
+
+  public int interpret(int[] in, int[] par) {    
+    return(exp.interpret(in,par)); 
+  }
+
+  public int arity() { return(arity); }
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Tdekllist.java b/tools/lexer/jflex-1.4/examples/interpreter/Tdekllist.java
new file mode 100644 (file)
index 0000000..6f409a6
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Tdekllist.java
@@ -0,0 +1,69 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for declaration lists of functions
+ */ 
+class Tdekllist implements AST {
+  Tdekllist dekllist;         // rest list (optional null)
+  Tdekl dekl;                 // declaration
+
+  public Tdekllist(Tdekllist p, Tdekl e) {
+    dekllist=p;
+    dekl=e;
+  }
+
+  public Tdekllist(Tdekl e) {
+    dekllist=null;
+    dekl=e;
+  }
+
+  public String toString() {
+    if (dekllist!=null) 
+      return(dekllist+",\n"+dekl);
+    else 
+      return(dekl.toString());
+  }
+
+  public void setSymtab(SymTab st) {
+    if (dekllist!=null) 
+      dekllist.setSymtab(st);
+    dekl.setSymtab(st);
+  }
+
+  public void printSymtabs() {
+    if (dekllist!=null) 
+      dekllist.printSymtabs();
+    dekl.printSymtabs();
+  }
+  
+  public void checkcontext() {
+    if (dekllist!=null) 
+      dekllist.checkcontext();    
+    dekl.checkcontext();              // CoCo (DefFun,DefVar,Arity)
+  }                                   // in function body
+
+  public void prepInterp(SymTab st) {  // set pointers and indices
+    dekl.prepInterp(st);
+    if (dekllist!=null) dekllist.prepInterp(st);
+  }
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Texp.java b/tools/lexer/jflex-1.4/examples/interpreter/Texp.java
new file mode 100644 (file)
index 0000000..8c4ef94
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Texp.java
@@ -0,0 +1,43 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for an integer expression.
+ * 
+ * The non terminal exp is the sum of multiple variants and
+ * therefore modeled as an abstract class.
+ * 
+ * The interpretation function <tt>interpret</tt> is called with
+ * valuations of input variables <tt>in</tt> and parameters
+ * <tt>par</tt>. Before interpret can be called, pointers
+ * and variable indices must be set with <tt>prepInterp</tt>.
+ */ 
+abstract class Texp implements AST {
+  // test context conditions (DefFun,DefVar,Arity)
+  abstract public void checkcontext(SymTab st);
+  
+  // set pointers and indices for variables and functions
+  abstract public void prepInterp(SymTab st);
+  
+  // interpretation
+  abstract public int interpret(int[] in, int[] par);
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Texpinfix.java b/tools/lexer/jflex-1.4/examples/interpreter/Texpinfix.java
new file mode 100644 (file)
index 0000000..9db0b0c
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Texpinfix.java
@@ -0,0 +1,64 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for infix expressions
+ */ 
+class Texpinfix extends Texp implements AST {
+  Texp exp1, exp2;            // left and right sub expression
+  char kind;                  // kind ('+', '-', '*', '/')
+
+  public Texpinfix(Texp e1, char k, Texp e2) {
+    exp1=e1;
+    kind=k;
+    exp2=e2;
+  }
+
+  public String toString() {
+    return("("+exp1+kind+exp2+")"); 
+  }
+
+  public void checkcontext(SymTab st) { // context conditions
+    exp1.checkcontext(st);
+    exp2.checkcontext(st);
+  }
+
+  public void prepInterp(SymTab st) {  // set pointers und indices
+    exp1.prepInterp(st);
+    exp2.prepInterp(st);
+  }
+
+  public int interpret(int[] in, int[] par) {    
+    int e1 = exp1.interpret(in,par);
+    int e2 = exp2.interpret(in,par);
+
+    switch (kind) {
+    case '+': return(e1+e2);
+    case '-': return(e1-e2);
+    case '*': return(e1*e2);
+    case '/': return(e1/e2);
+    }
+
+    return -1;        // error
+  }
+}
+
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Texplist.java b/tools/lexer/jflex-1.4/examples/interpreter/Texplist.java
new file mode 100644 (file)
index 0000000..84ffe96
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Texplist.java
@@ -0,0 +1,73 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for a list of expressions.
+ * 
+ * The interpretation of a list of expressions stores the
+ * results of the expressions in an array that can be used
+ * as parameter list for function calls.
+ */ 
+class Texplist implements AST {
+  Texplist explist;           // next list element (optional null)
+  Texp exp;                   // expression of this list node
+
+  public Texplist(Texplist p, Texp e) {
+    explist=p;
+    exp=e;
+  }
+
+  public Texplist(Texp e) {
+    explist=null;
+    exp=e;
+  }
+
+  public String toString() {
+    if (explist!=null)
+      return explist+","+exp;
+    else 
+      return exp.toString();
+  }
+
+  public void checkcontext(SymTab st) {
+    if (explist!=null) 
+      explist.checkcontext(st);
+    exp.checkcontext(st);             // CoCo (DefFun,DefVar,Arity)
+  }                                   // in expression
+
+  public int length() {
+    if (explist!=null) 
+      return 1+explist.length();
+    else 
+      return 1;
+  }
+  
+  public void prepInterp(SymTab st) {  // set pointers and indices
+    exp.prepInterp(st);
+    if (explist!=null) explist.prepInterp(st);
+  }
+  
+  public void interpret(int[] in, int[] par, int[] res, int index) {
+    res[index] = exp.interpret(in,par);
+    if (explist!=null) explist.interpret(in,par,res,index+1);
+  }
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Tfun.java b/tools/lexer/jflex-1.4/examples/interpreter/Tfun.java
new file mode 100644 (file)
index 0000000..9776ae9
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Tfun.java
@@ -0,0 +1,66 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for function application.
+ * 
+ * Also contains pointer to declaration location of the function.
+ */ 
+class Tfun extends Texp implements AST {
+  Tident ident;                       // name of the function
+  Texplist explist;                   // parameter list
+
+  public Tfun(Tident i, Texplist e) {
+    ident=i;
+    explist=e;
+  }
+
+  public String toString() {
+    return ident+"("+explist+")"; 
+  }
+
+  public void checkcontext(SymTab st) { // CoCo (DefFun,Arity)
+    explist.checkcontext(st);
+    SymtabEntry ste = st.lookup(ident.toString());
+    if (ste==null)
+      Main.error("function not defined: "+ident);
+    else if (ste.kind() != SymtabEntry.FUN)
+      Main.error("variable used as funktion: "+ident);
+    else if (((STEfun)ste).arity() != explist.length())
+      Main.error("wrong arity at function call: "+ident);
+  }
+
+  Tdekl fundekl;              // pointer to location of function declaration
+
+  // set pointers and indices 
+  public void prepInterp(SymTab st) {
+    fundekl = ((STEfun)st.lookup(ident.toString())).getDekl();
+    explist.prepInterp(st);
+  }
+
+  public int interpret(int[] in, int[] par) {
+    int[] newparams = new int[fundekl.arity()];
+    explist.interpret(in,par,newparams,0);
+    return fundekl.interpret(in,newparams);
+  }
+}
+
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Tident.java b/tools/lexer/jflex-1.4/examples/interpreter/Tident.java
new file mode 100644 (file)
index 0000000..dc8cba9
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Tident.java
@@ -0,0 +1,62 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for an identifier
+ */ 
+class Tident extends Texp implements AST {
+  String name;                
+
+  public Tident(String s) {
+    name = s; 
+  }
+
+  public String toString() {
+    return name; 
+  }
+
+  public void checkcontext(SymTab st) {         // CoCo (DefVar)
+    SymtabEntry ste = st.lookup(name);
+
+    if (ste==null)
+      Main.error("variable not defined: "+name);
+    else if (ste.kind() != SymtabEntry.VAR)
+      Main.error("function used as variable: "+name);
+  }
+
+  int index;              // number of ident in environment
+  boolean is_input;       // is it an input variable?    
+
+  public void prepInterp(SymTab st) {  // set index for environment 
+    STEvar ste = (STEvar)st.lookup(name);
+    index = ste.getIndex();
+    is_input = ste.isInput();
+  }
+
+  public int interpret(int[] in, int[] par) {
+    if (is_input) 
+      return(in[index]); 
+    else 
+      return(par[index]);
+  }
+}
+
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Tifthenelse.java b/tools/lexer/jflex-1.4/examples/interpreter/Tifthenelse.java
new file mode 100644 (file)
index 0000000..e7193f8
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Tifthenelse.java
@@ -0,0 +1,59 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for if-then-else expressions
+ */ 
+class Tifthenelse extends Texp implements AST {
+  Tboolexp boolexp;           // condition
+  Texp exp1, exp2;            // then and else branch
+  
+  public Tifthenelse(Tboolexp b, Texp e1, Texp e2) {
+    boolexp=b;
+    exp1=e1;
+    exp2=e2;
+  }
+  
+  public String toString() {
+    return "if "+boolexp+" then "+exp1+" else "+exp2+" fi"; 
+  }
+
+  public void checkcontext(SymTab st) {
+    boolexp.checkcontext(st);
+    exp1.checkcontext(st);
+    exp2.checkcontext(st);
+  }
+
+  public void prepInterp(SymTab st) {
+    boolexp.prepInterp(st);
+    exp1.prepInterp(st);
+    exp2.prepInterp(st);
+  }
+
+  public int interpret(int[] in, int[] par) {
+    boolean b = boolexp.interpret(in,par);
+    if (b) 
+      return exp1.interpret(in,par);
+    else 
+      return exp2.interpret(in,par);
+  }
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Tnumber.java b/tools/lexer/jflex-1.4/examples/interpreter/Tnumber.java
new file mode 100644 (file)
index 0000000..fef710a
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Tnumber.java
@@ -0,0 +1,47 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for a number
+ */ 
+class Tnumber extends Texp implements AST {
+  int n;             // value of the number
+
+  public Tnumber(String s) {
+    try { n = Integer.parseInt(s); }
+    catch (NumberFormatException e) { n=-1; };
+  }
+
+  public String toString() {
+    return(""+n); 
+  }
+
+  public void checkcontext(SymTab st) { 
+  }
+
+  public void prepInterp(SymTab st) { 
+  }
+
+  public int interpret(int[] in, int[] par) {
+    return(n);
+  }
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Tparlist.java b/tools/lexer/jflex-1.4/examples/interpreter/Tparlist.java
new file mode 100644 (file)
index 0000000..6f2ada9
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Tparlist.java
@@ -0,0 +1,54 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for a parameter list.
+ */ 
+class Tparlist implements AST {
+  Tparlist parlist;           // rest of the liste (optional null)
+  Tident ident;               // identifier
+
+  public Tparlist(Tparlist p, Tident i) {
+    parlist=p;
+    ident=i;
+  }
+
+  public Tparlist(Tident i) {
+    parlist=null;
+    ident=i;
+  }
+
+  public String toString() {
+    if (parlist!=null) 
+      return parlist+","+ident;
+    else 
+      return ident.toString();
+  }
+
+  public void setSymtab(SymTab st, boolean isInput, int index) {
+    boolean isNew = st.enter(ident.toString(),
+                             new STEvar(ident.toString(), isInput, index));
+    
+    if (!isNew) Main.error("Variable "+ident+" defined twice!");
+    if (parlist!=null) parlist.setSymtab(st, isInput, index+1);
+  }
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Tprogram.java b/tools/lexer/jflex-1.4/examples/interpreter/Tprogram.java
new file mode 100644 (file)
index 0000000..08a93d1
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Tprogram.java
@@ -0,0 +1,99 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for the whole program (top node).
+ * 
+ * Also contains two symbol tables, one for input variables,
+ * one for function names. 
+ *
+ * All operations like context check, symbol table build up
+ * etc. start here.
+ */ 
+class Tprogram implements AST {
+
+  Tparlist parlist;           // input variables
+  Tdekllist dekllist;         // function declarations 
+  Texplist explist;           // result expressions
+  Texplist arguments;         // input values  
+
+  public Tprogram(Tparlist p, Tdekllist d, Texplist e, Texplist a) {
+    parlist=p;
+    dekllist=d;
+    explist=e;
+    arguments=a;
+  }
+
+  public String toString() {
+    return("Program:\n=============\ninput "+parlist+
+           "\nfunctions\n"+dekllist+"\noutput "+explist+
+           "\narguments "+arguments+"\nend");
+  }
+
+  SymTab inputs;      // table of input variables
+  SymTab functions;   // table of functions
+
+  public void setSymtabs() {          // calculate symbol table entries
+    inputs = new SymTab();            // set input variables
+    parlist.setSymtab(inputs, true, 0);
+    functions = new SymTab(inputs);
+    dekllist.setSymtab(functions);
+  }
+
+  public void printSymtabs() {
+    System.out.print("Input variables-\n"+inputs);
+    System.out.print("Functions-\n"+functions);
+    dekllist.printSymtabs();
+  }
+
+  public void checkcontext() {
+    dekllist.checkcontext();          // CoCo (DefFun,DefVar,Arity)
+                                      // in function bodies
+    explist.checkcontext(functions);  // CoCo (DefFun,DefVar,Arity)
+                                      // in result expressions
+    arguments.checkcontext(new SymTab()); // CoCo  (constants)
+                                          // in arguments
+    if (arguments.length()!=inputs.size())
+      Main.error("Argument list and input variables list differ!");
+  }
+
+  public void prepInterp() {          // set pointers and indices
+    dekllist.prepInterp(functions);
+    explist.prepInterp(functions);
+  }
+  
+  public void interpret() {    
+    int[] inputEnv = new int[inputs.size()];      // set input 
+
+    arguments.interpret(null,null,inputEnv,0);
+
+    System.out.println("Result:\n=============");
+
+    int[] ergebnis = new int[explist.length()];
+    explist.interpret(inputEnv,null,ergebnis,0);  // calculate result
+
+    int i;
+    for (i=explist.length()-1; i > 0; i--)
+      System.out.print(ergebnis[i]+",  ");
+    System.out.println(ergebnis[i]);
+  }
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/Tuminus.java b/tools/lexer/jflex-1.4/examples/interpreter/Tuminus.java
new file mode 100644 (file)
index 0000000..d269b34
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/Tuminus.java
@@ -0,0 +1,49 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+/**
+ * AST node for unary minus expressions.
+ */ 
+class Tuminus extends Texp implements AST {
+
+  Texp exp;                           // the negated expression
+
+  public Tuminus(Texp e) {
+    exp=e; 
+  }
+
+  public String toString() {
+    return "-"+exp; 
+  }
+
+  public void checkcontext(SymTab st) {
+    exp.checkcontext(st); 
+  }
+
+  public void prepInterp(SymTab st) {  
+    exp.prepInterp(st); 
+  }
+
+  public int interpret(int[] in, int[] par) {
+    return -(exp.interpret(in,par)); 
+  }
+}
+

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/build.xml b/tools/lexer/jflex-1.4/examples/interpreter/build.xml
new file mode 100644 (file)
index 0000000..51829ba
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/build.xml
@@ -0,0 +1,38 @@
+<project name="interpreter" default="run">
+
+  <taskdef classname="JFlex.anttask.JFlexTask" name="jflex" />
+  <taskdef classname="java_cup.AntTask" name="cup" />
+
+  <target name="build" depends="realclean,compile" />
+
+  <target name="run" depends="compile">
+    <java classname="Main"> <arg line="example.as"/> </java>
+  </target>
+
+  <target name="compile" depends="jflex,cup">
+    <javac srcdir="." destdir="." />
+  </target>
+
+  <target name="jflex">
+    <jflex file="scanner.flex" />
+  </target>
+
+  <target name="cup">
+    <cup file="parser.cup" interface="yes" dir="."/>
+  </target>
+
+  <target name="genclean">
+    <delete file="Yylex.java"/>
+    <delete file="sym.java" />
+    <delete file="parser.java" />
+  </target>
+
+  <target name="clean">
+    <delete>
+      <fileset dir="." includes="**/*~"/>
+      <fileset dir="." includes="**/*.class"/>
+    </delete>
+  </target>
+
+  <target name="realclean" depends="clean,genclean" />
+</project>

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/example.as b/tools/lexer/jflex-1.4/examples/interpreter/example.as
new file mode 100644 (file)
index 0000000..8152247
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/example.as
@@ -0,0 +1,16 @@
+
+// example program for interpreter testing 
+// contains division and modulo functins 
+
+input a,b
+functions div(x,y) = if x < y
+                     then 0
+                     else div(x-y,y)+1
+                     fi,
+          mod(x,y) = if x < y
+                     then x
+                     else mod(x-y,y)
+                     fi
+output div(a,b), mod(a,b)
+arguments 324, 17
+end

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/parser.cup b/tools/lexer/jflex-1.4/examples/interpreter/parser.cup
new file mode 100644 (file)
index 0000000..3d7f856
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/parser.cup
@@ -0,0 +1,112 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+// definition of tokens, if applicable with token type 
+terminal INPUT, FUNCTIONS, OUTPUT, END, ARGUMENTS;
+terminal IF, THEN, ELSE, FI, ERROR;
+terminal COMMA, LPAR, RPAR;
+terminal EQ, LE, LEQ, MINUS, PLUS, TIMES, DIV, UMINUS;
+
+terminal String ID, NUMBER;
+
+non terminal Tprogram  program;
+non terminal Tparlist  parlist;
+non terminal Texplist  explist;
+non terminal Tdekllist dekllist;
+non terminal Tdekl     dekl;
+non terminal Texp      exp;
+non terminal Tboolexp  boolexp;
+non terminal Tident    ident;
+non terminal Tnumber   number;
+
+
+// precedences, left associativity
+precedence left EQ, LE, LEQ;
+precedence left MINUS, PLUS;
+precedence left TIMES, DIV;
+precedence left UMINUS;
+
+
+// here the rules start 
+program       ::= INPUT parlist:p FUNCTIONS dekllist:d OUTPUT explist:o
+                                        ARGUMENTS explist:a END
+                  {: RESULT = new Tprogram(p,d,o,a); :}
+                ;
+
+parlist       ::= ident:i
+                  {: RESULT = new Tparlist(i); :}
+                | parlist:p COMMA ident:i
+                  {: RESULT = new Tparlist(p,i); :}
+                ;
+
+explist       ::= exp:e
+                  {: RESULT = new Texplist(e); :}
+                | explist:l COMMA exp:e
+                  {: RESULT = new Texplist(l,e); :}
+                ;
+
+dekllist      ::= dekl:d
+                  {: RESULT = new Tdekllist(d);:}
+                | dekllist:l COMMA dekl:d
+                  {: RESULT = new Tdekllist(l,d); :}
+                ;
+
+dekl          ::= ident:i LPAR parlist:p RPAR EQ exp:e
+                  {: RESULT = new Tdekl(i,p,e); :}
+                ;
+
+exp           ::= number:n
+                  {: RESULT = n; :}
+                | ident:i
+                  {: RESULT = i; :}
+                | ident:i LPAR explist:e RPAR
+                  {: RESULT = new Tfun(i,e); :}
+                | LPAR exp:e RPAR
+                  {: RESULT = e; :}
+                | MINUS exp:e                
+                  {: RESULT = new Tuminus(e); :} %prec UMINUS
+                | exp:l PLUS exp:r
+                  {: RESULT = new Texpinfix(l,'+',r); :}
+                | exp:l TIMES exp:r
+                  {: RESULT = new Texpinfix(l,'*',r); :}
+                | exp:l DIV exp:r
+                  {: RESULT = new Texpinfix(l,'/',r); :}
+                | exp:l MINUS exp:r
+                  {: RESULT = new Texpinfix(l,'-',r); :}
+                | IF boolexp:b THEN exp:t ELSE exp:e FI
+                  {: RESULT = new Tifthenelse(b,t,e); :}
+                ;
+
+boolexp       ::= exp:l EQ exp:r
+                  {: RESULT = new Tboolexp(l,'=',r); :}
+                | exp:l LE exp:r
+                  {: RESULT = new Tboolexp(l,'<',r); :}
+                | exp:l LEQ exp:r
+                  {: RESULT = new Tboolexp(l,'!',r); :}
+                ;
+
+ident         ::= ID:n
+                  {: RESULT = new Tident(n); :}
+                ;
+
+number        ::= NUMBER:z
+                  {: RESULT = new Tnumber(z); :}
+                ;

diff --git a/tools/lexer/jflex-1.4/examples/interpreter/scanner.flex b/tools/lexer/jflex-1.4/examples/interpreter/scanner.flex
new file mode 100644 (file)
index 0000000..a6c3ca7
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/interpreter/scanner.flex
@@ -0,0 +1,71 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2001       Gerwin Klein <lsf@jflex.de>                    *
+ * Copyright (C) 2001       Bernhard Rumpe <rumpe@in.tum.de>               *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+import java_cup.runtime.Symbol;
+
+%%
+
+%cup
+%implements sym
+
+%{ 
+  SymTab symtab;          // externe symbol table
+
+  public void setSymtab(SymTab symtab) {
+    this.symtab = symtab; 
+  }
+
+  private Symbol sym(int sym) {
+    return new Symbol(sym);
+  }
+
+  private Symbol sym(int sym, Object val) {
+    return new Symbol(sym, val);
+  }
+%}
+
+%%
+
+"arguments"     { return sym(ARGUMENTS); }
+"input"         { return sym(INPUT); }
+"functions"     { return sym(FUNCTIONS); }
+"output"        { return sym(OUTPUT); }
+"end"           { return sym(END); }
+"if"            { return sym(IF); }
+"then"          { return sym(THEN); }
+"else"          { return sym(ELSE); }
+"fi"            { return sym(FI); }
+[a-z]+          { symtab.enter(yytext(),new SymtabEntry(yytext()));
+                  return sym(ID,yytext()); }
+[0-9]+          { return sym(NUMBER,yytext()); }
+","             { return sym(COMMA); }
+"("             { return sym(LPAR); }
+")"             { return sym(RPAR); }
+"="             { return sym(EQ); }
+"-"             { return sym(MINUS); }
+"+"             { return sym(PLUS); }
+"*"             { return sym(TIMES); }
+"/"             { return sym(DIV); }
+"<"             { return sym(LE); }
+"<="            { return sym(LEQ); }
+[\ \t\b\f\r\n]+ { /* eat whitespace */ }
+"//"[^\n]*      { /* one-line comment */ }
+.               { throw new Error("Unexpected character ["+yytext()+"]"); }

diff --git a/tools/lexer/jflex-1.4/examples/java/JavaParser.java b/tools/lexer/jflex-1.4/examples/java/JavaParser.java
new file mode 100644 (file)
index 0000000..4c58705
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/java/JavaParser.java
@@ -0,0 +1,47 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 1998-2004  Gerwin Klein <lsf@jflex.de>                    *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+import java.io.FileReader;
+
+
+/**
+ * Simple test driver for the java parser. Just runs it on some
+ * input files, gives no useful output.
+ */
+public class JavaParser {
+
+  public static void main(String argv[]) {
+
+    for (int i = 0; i < argv.length; i++) {
+      try {
+        System.out.println("Parsing ["+argv[i]+"]");
+        Scanner s = new Scanner(new UnicodeEscapes(new FileReader(argv[i])));
+        parser p = new parser(s);
+        p.parse();
+        
+        System.out.println("No errors.");
+      }
+      catch (Exception e) {
+        e.printStackTrace(System.out);
+        System.exit(1);
+      }
+    }
+  }
+
+}

diff --git a/tools/lexer/jflex-1.4/examples/java/JavaSymbol.java b/tools/lexer/jflex-1.4/examples/java/JavaSymbol.java
new file mode 100644 (file)
index 0000000..d27b63d
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/java/JavaSymbol.java
@@ -0,0 +1,31 @@
+
+public class JavaSymbol extends java_cup.runtime.Symbol {
+  private int line;
+  private int column;
+
+  public JavaSymbol(int type, int line, int column) {
+    this(type, line, column, -1, -1, null);
+  }
+
+  public JavaSymbol(int type, int line, int column, Object value) {
+    this(type, line, column, -1, -1, value);
+  }
+
+  public JavaSymbol(int type, int line, int column, int left, int right, Object value) {
+    super(type, left, right, value);
+    this.line = line;
+    this.column = column;
+  }
+
+  public int getLine() {
+    return line;
+  }
+
+  public int getColumn() {
+    return column;
+  }
+
+  public String toString() {   
+    return "line "+line+", column "+column+", sym: "+sym+(value == null ? "" : (", value: '"+value+"'"));
+  }
+}

diff --git a/tools/lexer/jflex-1.4/examples/java/Makefile b/tools/lexer/jflex-1.4/examples/java/Makefile
new file mode 100644 (file)
index 0000000..655ebd3
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/java/Makefile
@@ -0,0 +1,48 @@
+# You need CUP v0.10j (or newer) for this makefile to work (for java12.cup)
+#
+# CUP classes should be included in CLASSPATH 
+
+CUP        = java java_cup.Main -interface <
+JFLEX      = jflex
+JAVA       = java
+JAVAC      = javac
+#JAVAC      = jikes 
+JAVACFLAGS = 
+
+# --------------------------------------------------
+
+all: test
+
+test: lexer-output.txt
+       @(diff lexer-output.txt lexer-output.good && echo "Test OK!") || echo "Test failed!"
+
+lexer-output.txt: compile
+       $(JAVA) TestLexer TestLexer.java > lexer-output.txt
+
+
+compile: scanner parser unicode
+       $(JAVAC) $(JAVACFLAGS) JavaParser.java TestLexer.java
+
+parser: parser.java 
+
+parser.java: java12.cup
+       $(CUP) java12.cup
+
+scanner: Scanner.java
+
+Scanner.java: java.flex
+       $(JFLEX) java.flex
+
+unicode: UnicodeEscapes.java
+
+UnicodeEscapes.java: unicode.flex
+       $(JFLEX) unicode.flex
+
+clean:
+       rm -f *.class
+       rm -f *~
+       rm -f Scanner.java
+       rm -f parser.java
+       rm -f sym.java
+       rm -f UnicodeEscapes.java
+       rm -f lexer-output.txt

diff --git a/tools/lexer/jflex-1.4/examples/java/README b/tools/lexer/jflex-1.4/examples/java/README
new file mode 100644 (file)
index 0000000..096582b
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/java/README
@@ -0,0 +1,41 @@
+This directory contains a scanner and parser for the Java programming language.
+
+You need the parser generator CUP v0.10j (or newer) for the parser to work.
+Use the Makefile to generate the lexer and parser, or type:
+
+jflex unicode.flex
+jflex java.flex
+java java_cup.Main -interface < java12.cup
+javac JavaParser.java TestLexer.java
+
+
+The parser can be tested with:
+java JavaParser <inputfiles>
+
+The scanner (without parser attached) can be test with:
+java TestLexer <inputfiles>
+
+
+files:
+
+unicode.flex
+  JFlex specification for the Unicode preprocessing phase
+  (see section 3.3 of the Java Language Specification).
+  Demonstrates how to implement a FilterReader with JFlex.
+
+java.flex
+  JFlex specification for the "real" Java 1.2 lexer.
+
+java12.cup
+  CUP specification for the Java 1.2 parser
+  Copyright (C) 1998 by C. Scott Ananian <cananian@alumni.princeton.edu>
+  (with small modifications to interface with the Lexer)
+
+JavaParser.java
+  a simple main class to run the parser (no other useful output though)
+
+TestLexer.java
+  a simple test driver for the scanner, produces debug output
+
+lexer-output.good
+  lexer output for input file TestLexer.java for testing

diff --git a/tools/lexer/jflex-1.4/examples/java/TestLexer.java b/tools/lexer/jflex-1.4/examples/java/TestLexer.java
new file mode 100644 (file)
index 0000000..90e4f35
--- /dev/null
+++ b/tools/lexer/jflex-1.4/examples/java/TestLexer.java
@@ -0,0 +1,62 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright (C) 2004       Gerwin Klein <lsf@jflex.de>                    *
+ * All rights reserved.                                                    *
+ *                                                                         *
+ * This program is free software; you can redistribute it and/or modify    *
+ * it under the terms of the GNU General Public License. See the file      *
+ * COPYRIGHT for more information.                                         *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful,         *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+ * GNU General Public License for more details.                            *
+ *                                                                         *
+ * You should have received a copy of the GNU General Public License along *
+ * with this program; if not, write to the Free Software Foundation, Inc., *
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA                 *
+ *                                                                         *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+import java_cup.runtime.Symbol;
+
+import java.io.FileReader;
+
+
+/**
+ * Simple test driver for the java lexer. Just runs it on some
+ * input files and produces debug output. Needs symbol class from
+ * parser. 
+ */
+public class TestLexer {
+
+  /** some numerals to for lexer testing */
+  int intDec = 37;
+  long longDec = 37l;
+  int intHex = 0x0001;
+  long longHex = 0xFFFFl;
+  int intOct = 0377;
+  long longOc = 007l;
+   
+
+  public static void main(String argv[]) {
+
+    for (int i = 0; i < argv.length; i++) {
+      try {
+        System.out.println("Lexing ["+argv[i]+"]");
+        Scanner scanner = new Scanner(new UnicodeEscapes(new FileReader(argv[i])));
+                
+