java/io/StreamTokenizer.java

[Oskar Liljeblad <osk at hem dot passagen dot se>]

I forgot to attach the patch.
--- StreamTokenizer_gcj.java	Sun Jul 23 00:56:19 2000
+++ StreamTokenizer.java	Sun Jul 23 01:40:52 2000
@@ -9,6 +9,10 @@
 package java.io;
 
 /**
+ * This class parses streams of characters into tokens.  There are a
+ * million-zillion flags that can be set to control the parsing, as 
+ * described under the various method headings.
+ *
  * @author Warren Levy <warrenl@cygnus.com>
  * @date October 25, 1998.  
  */
@@ -20,25 +24,40 @@
  
 public class StreamTokenizer
 {
-  /* A constant indicating that the end of the stream has been read. */
+  /** A constant indicating that the end of the stream has been read. */
   public static final int TT_EOF = -1;
 
-  /* A constant indicating that the end of the line has been read. */
+  /** A constant indicating that the end of the line has been read. */
   public static final int TT_EOL = '\n';
 
-  /* A constant indicating that a number token has been read. */
+  /** A constant indicating that a number token has been read. */
   public static final int TT_NUMBER = -2;
 
-  /* A constant indicating that a word token has been read. */
+  /** A constant indicating that a word token has been read. */
   public static final int TT_WORD = -3;
 
-  /* Contains the type of the token read resulting from a call to nextToken. */
-  public int ttype;
+  /** A constant indicating that no tokens have been read yet. */
+  public static final int TT_NONE = -4;
 
-  /* The String associated with word and string tokens. */
+  /**
+   * Contains the type of the token read resulting from a call to nextToken
+   * The rules are as follows:
+   * <ul>
+   * <li>For a token consisting of a single ordinary character, this is the 
+   *     value of that character.
+   * <li>For a quoted string, this is the value of the quote character
+   * <li>For a word, this is TT_WORD
+   * <li>For a number, this is TT_NUMBER
+   * <li>For the end of the line, this is TT_EOL
+   * <li>For the end of the stream, this is TT_EOF
+   * </ul>
+   */
+  public int ttype = TT_NONE;
+
+  /** The String associated with word and string tokens. */
   public String sval;
 
-  /* The numeric value associated with number tokens. */
+  /** The numeric value associated with number tokens. */
   public double nval;
 
   /* Indicates whether end-of-line is recognized as a token. */
@@ -54,11 +73,11 @@
   private boolean slashStar = false;
 
   /* Attribute tables of each byte from 0x00 to 0xFF. */
-  private boolean[] whitespace;
-  private boolean[] alphabetic;
-  private boolean[] numeric;
-  private boolean[] quote;
-  private boolean[] comment;
+  private boolean[] whitespace = new boolean[256];
+  private boolean[] alphabetic = new boolean[256];
+  private boolean[] numeric = new boolean[256];
+  private boolean[] quote = new boolean[256];
+  private boolean[] comment = new boolean[256];
 
   /* The Reader associated with this class. */
   private PushbackReader in;
@@ -69,24 +88,45 @@
   /* Contains the current line number of the reader. */
   private int lineNumber = 1;
 
-  // Deprecated in JDK 1.1.
+  /**
+   * This method reads bytes from an <code>InputStream</code> and tokenizes
+   * them.  For details on how this method operates by default, see
+   * <code>StreamTokenizer(Reader)</code>.
+   *
+   * @param in The <code>InputStream</code> to read from
+   *
+   * @deprecated Since JDK 1.1.
+   */
   public StreamTokenizer(InputStream is)
   {
     this(new InputStreamReader(is));
   }
 
+  /**
+   * This method initializes a new <code>StreamTokenizer</code> to read 
+   * characters from a <code>Reader</code> and parse them.  The char values
+   * have their hight bits masked so that the value is treated a character
+   * in the range of 0x0000 to 0x00FF.
+   * <p>
+   * This constructor sets up the parsing table to parse the stream in the
+   * following manner:
+   * <ul>
+   * <li>The values 'A' through 'Z', 'a' through 'z' and 0xA0 through 0xFF
+   *     are initialized as alphabetic
+   * <li>The values 0x00 through 0x20 are initialized as whitespace
+   * <li>The values '\'' and '"' are initialized as quote characters
+   * <li>'/' is a comment character
+   * <li>Numbers will be parsed
+   * <li>EOL is not treated as significant
+   * <li>C  and C++ (//) comments are not recognized
+   * </ul>
+   *
+   * @param in The <code>Reader</code> to read chars from
+   */
   public StreamTokenizer(Reader r)
   {
     in = new PushbackReader(r);
 
-    whitespace = new boolean[256];
-    alphabetic = new boolean[256];
-    numeric = new boolean[256];
-    quote = new boolean[256];
-    comment = new boolean[256];
-    for (int i = 0; i < 256; i++)
-      resetChar(i);
-
     whitespaceChars(0x00, 0x20);
     wordChars('A', 'Z');
     wordChars('a', 'z');
@@ -97,22 +137,48 @@
     parseNumbers();
   }
 
+  /**
+   * This method sets the comment attribute on the specified character.
+   *
+   * @param c The character to set the comment attribute for, passed as an int
+   */
   public void commentChar(int ch)
   {
     if (ch >= 0 && ch <= 255)
       comment[ch] = true;
   }
 
+  /**
+   * This method sets a flag that indicates whether or not the end of line
+   * sequence terminates and is a token.  The defaults to <code>false</code>
+   *
+   * @param flag <code>true</code> if EOF is significant, <code>false</code>
+   *             otherwise
+   */
   public void eolIsSignificant(boolean flag)
   {
     eolSignificant = flag;
   }
 
+  /**
+   * This method returns the current line number.  Note that if the 
+   * <code>pushBack()</code> method is called, it has no effect on the
+   * line number returned by this method.
+   *
+   * @return The current line number
+   */
   public int lineno()
   {
     return lineNumber;
   }
 
+  /**
+   * This method sets a flag that indicates whether or not alphabetic
+   * tokens that are returned should be converted to lower case.
+   * 
+   * @param flag <code>true</code> to convert to lower case,
+   *             <code>false</code> otherwise
+   */
   public void lowerCaseMode(boolean flag)
   {
     lowerCase = flag;
@@ -120,52 +186,88 @@
 
   private boolean isWhitespace(int ch)
   {
-    if (ch >= 0 && ch <= 255)
-      return whitespace[ch];
-
-    return false;
+    return (ch >= 0 && ch <= 255 && whitespace[ch]);
   }
 
   private boolean isAlphabetic(int ch)
   {
-    if (ch >= 0 && ch <= 255)
-      return alphabetic[ch];
-    else if (ch > 255)
-      return true;
-
-    return false;
+    return ((ch > 255) || (ch >= 0 && alphabetic[ch]));
   }
 
   private boolean isNumeric(int ch)
   {
-    if (ch >= 0 && ch <= 255)
-      return numeric[ch];
-
-    return false;
+    return (ch >= 0 && ch <= 255 && numeric[ch]);
   }
 
   private boolean isQuote(int ch)
   {
-    if (ch >= 0 && ch <= 255)
-      return quote[ch];
-
-    return false;
+    return (ch >= 0 && ch <= 255 && quote[ch]);
   }
 
   private boolean isComment(int ch)
   {
-    if (ch >= 0 && ch <= 255)
-      return comment[ch];
-
-    return false;
+    return (ch >= 0 && ch <= 255 && comment[ch]);
   }
 
+  /**
+   * This method reads the next token from the stream.  It sets the 
+   * <code>ttype</code> variable to the appropriate token type and 
+   * returns it.  It also can set <code>sval</code> or <code>nval</code>
+   * as described below.  The parsing strategy is as follows:
+   * <ul>
+   * <li>Skip any whitespace characters.
+   * <li>If a numeric character is encountered, attempt to parse a numeric
+   * value.  Leading '-' characters indicate a numeric only if followed by
+   * another non-'-' numeric.  The value of the numeric token is terminated
+   * by either the first non-numeric encountered, or the second occurrence of
+   * '-' or '.'.  The token type returned is TT_NUMBER and <code>nval</code>
+   * is set to the value parsed.
+   * <li>If an alphabetic character is parsed, all subsequent characters
+   * are read until the first non-alphabetic or non-numeric character is
+   * encountered.  The token type returned is TT_WORD and the value parsed
+   * is stored in <code>sval</code>.  If lower case mode is set, the token
+   * stored in <code>sval</code> is converted to lower case.  The end of line
+   * sequence terminates a word only if EOL signficance has been turned on.
+   * The start of a comment also terminates a word.  Any character with a 
+   * non-alphabetic and non-numeric attribute (such as white space, a quote,
+   * or a commet) are treated as non-alphabetic and terminate the word.
+   * <li>If a comment charcters is parsed, then all remaining characters on
+   * the current line are skipped and another token is parsed.  Any EOL or
+   * EOF's encountered are not discarded, but rather terminate the comment.
+   * <li>If a quote character is parsed, then all characters up to the 
+   * second occurrence of the same quote character are parsed into a
+   * <code>String</code>.  This <code>String</code> is stored as
+   * <code>sval</code>, but is not converted to lower case, even if lower case
+   * mode is enabled.  The token type returned is the value of the quote
+   * character encountered.  Any escape sequences
+   * (\b (backspace), \t (HTAB), \n (linefeed), \f (form feed), \r
+   * (carriage return), \" (double quote), \' (single quote), \\
+   * (backslash), \XXX (octal esacpe)) are converted to the appropriate
+   * char values.  Invalid esacape sequences are left in untranslated.  
+   * Unicode characters like ('\ u0000') are not recognized. 
+   * <li>If the C++ comment sequence "//" is encountered, and the parser
+   * is configured to handle that sequence, then the remainder of the line
+   * is skipped and another token is read exactly as if a character with
+   * the comment attribute was encountered.
+   * <li>If the C comment sequence "/*" is encountered, and the parser
+   * is configured to handle that sequence, then all characters up to and
+   * including the comment terminator sequence are discarded and another
+   * token is parsed.
+   * <li>If all cases above are not met, then the character is an ordinary
+   * character that is parsed as a token by itself.  The char encountered
+   * is returned as the token type.
+   * </ul>
+   *
+   * @return The token type
+   * @exception IOException If an I/O error occurs
+   */
   public int nextToken() throws IOException
   {
     if (pushedBack)
       {
 	pushedBack = false;
-	return ttype;
+	if (ttype != TT_NONE)
+	  return ttype;
       }
 
     sval = null;
@@ -355,12 +457,32 @@
       false;
   }
 
+  /**
+   * This method makes the specified character an ordinary character.  This
+   * means that none of the attributes (whitespace, alphabetic, numeric,
+   * quote, or comment) will be set on this character.  This character will
+   * parse as its own token.
+   *
+   * @param c The charcter to make ordinary, passed as an int
+   */
   public void ordinaryChar(int ch)
   {
     if (ch >= 0 && ch <= 255)
       resetChar(ch);
   }
 
+  /**
+   * This method makes all the characters in the specified range, range
+   * terminators included, ordinary.  This means the none of the attributes
+   * (whitespace, alphabetic, numeric, quote, or comment) will be set on
+   * any of the characters in the range.  This makes each character in this
+   * range parse as its own token.
+   *
+   * @param low The low end of the range of values to set the whitespace
+   *            attribute for
+   * @param high The high end of the range of values to set the whitespace
+   *            attribute for
+   */
   public void ordinaryChars(int low, int hi)
   {
     if (low < 0)
@@ -371,6 +493,10 @@
       resetChar(i);
   }
 
+  /**
+   * This method sets the numeric attribute on the characters '0' - '9' and
+   * the characters '.' and '-'.
+   */
   public void parseNumbers()
   {
     for (int i = 0; i <= 9; i++)
@@ -380,6 +506,13 @@
     numeric['-'] = true;
   }
 
+  /**
+   * This method returns the current line number.  Note that if the 
+   * <code>pushBack()</code> method is called, it has no effect on the
+   * line number returned by this method.
+   *
+   * @return The current line number
+   */
   public void pushBack()
   {
     // pushBack may cause the lineno method to return an incorrect value
@@ -387,27 +520,74 @@
     pushedBack = true;
   }
 
+  /**
+   * This method sets the quote attribute on the specified character.
+   *
+   * @param c The character to set the quote attribute for, passed as an int.
+   */
   public void quoteChar(int ch)
   {
     if (ch >= 0 && ch <= 255)
       quote[ch] = true;
   }
 
+  /**
+   * This method removes all attributes (whitespace, alphabetic, numeric,
+   * quote, and comment) from all characters.  It is equivalent to calling
+   * <code>ordinaryChars(0x00, 0xFF)</code>.
+   *
+   * @see ordinaryChars
+   */
   public void resetSyntax()
   {
     ordinaryChars(0x00, 0xFF);
   }
 
+  /**
+   * This method sets a flag that indicates whether or not "C++" language style
+   * comments ("//" comments through EOL ) are handled by the parser.
+   * If this is <code>true</code> commented out sequences are skipped and
+   * ignored by the parser.  This defaults to <code>false</code>.
+   *
+   * @param flag <code>true</code> to recognized and handle "C++" style
+   *             comments, <code>false</code> otherwise
+   */
   public void slashSlashComments(boolean flag)
   {
     slashSlash = flag;
   }
 
+  /**
+   * This method sets a flag that indicates whether or not "C" language style
+   * comments (with nesting not allowed) are handled by the parser.
+   * If this is <code>true</code> commented out sequences are skipped and
+   * ignored by the parser.  This defaults to <code>false</code>.
+   *
+   * @param flag <code>true</code> to recognized and handle "C" style comments,
+   *             <code>false</code> otherwise
+   */
   public void slashStarComments(boolean flag)
   {
     slashStar = flag;
   }
 
+  /**
+   * This method returns the current token value as a <code>String</code> in
+   * the form "Token[x], line n", where 'n' is the current line numbers and
+   * 'x' is determined as follows.
+   * <p>
+   * <ul>
+   * <li>If no token has been read, then 'x' is "NOTHING" and 'n' is 0
+   * <li>If <code>ttype</code> is TT_EOF, then 'x' is "EOF"
+   * <li>If <code>ttype</code> is TT_EOL, then 'x' is "EOL"
+   * <li>If <code>ttype</code> is TT_WORD, then 'x' is <code>sval</code>
+   * <li>If <code>ttype</code> is TT_NUMBER, then 'x' is "n=strnval" where
+   * 'strnval' is <code>String.valueOf(nval)</code>.
+   * <li>If <code>ttype</code> is a quote character, then 'x' is
+   * <code>sval</code>
+   * <li>For all other cases, 'x' is <code>ttype</code>
+   * </ul>
+   */
   public String toString()
   {
     String tempstr;
@@ -418,13 +598,24 @@
     else if (ttype == TT_WORD)
       tempstr = sval;
     else if (ttype == TT_NUMBER)
-      tempstr = "n=" + Double.toString(nval);
+      tempstr = "n=" + nval;
+    else if (ttype == TT_NONE)
+      tempstr = "NOTHING";
     else // must be an ordinary char.
-      tempstr = "\'" + (new Character((char) ttype)).toString() + "\'";
+      tempstr = "\'" + (char) ttype + "\'";
 
-    return "Token[" + tempstr + "], line " + Integer.toString(lineno());
+    return "Token[" + tempstr + "], line " + lineno();
   }
 
+  /**
+   * This method sets the whitespace attribute for all charcters in the
+   * specified range, range terminators included.
+   *
+   * @param low The low end of the range of values to set the whitespace
+   *            attribute for
+   * @param high The high end of the range of values to set the whitespace
+   *             attribute for
+   */
   public void whitespaceChars(int low, int hi)
   {
     if (low < 0)
@@ -435,6 +626,15 @@
       whitespace[i] = true;
   }
 
+  /**
+   * This method sets the alphabetic attribute for all charcters in the
+   * specified range, range terminators included.
+   *
+   * @param low The low end of the range of values to set the alphabetic
+   *            attribute for
+   * @param high The high end of the range of values to set the alphabetic
+   *             attribute for
+   */
   public void wordChars(int low, int hi)
   {
     if (low < 0)