libjava/java/io/StreamTokenizer.java

   1 /* Copyright (C) 1998, 1999  Cygnus Solutions
   2
   3    This file is part of libgcj.
   4
   5 This software is copyrighted work licensed under the terms of the
   6 Libgcj License.  Please consult the file "LIBGCJ_LICENSE" for
   7 details.  */
   8
   9 package java.io;
  10
  11 /**
  12  * @author Warren Levy <warrenl@cygnus.com>
  13  * @date October 25, 1998.
  14  */
  15 /* Written using "Java Class Libraries", 2nd edition, ISBN 0-201-31002-3
  16  * "The Java Language Specification", ISBN 0-201-63451-1
  17  * plus online API docs for JDK 1.2 beta from http://www.javasoft.com.
  18  * Status:  Believed complete and correct.
  19  */
  20
  21 public class StreamTokenizer
  22 {
  23   /* A constant indicating that the end of the stream has been read. */
  24   public static final int TT_EOF = -1;
  25
  26   /* A constant indicating that the end of the line has been read. */
  27   public static final int TT_EOL = '\n';
  28
  29   /* A constant indicating that a number token has been read. */
  30   public static final int TT_NUMBER = -2;
  31
  32   /* A constant indicating that a word token has been read. */
  33   public static final int TT_WORD = -3;
  34
  35   /* Contains the type of the token read resulting from a call to nextToken. */
  36   public int ttype;
  37
  38   /* The String associated with word and string tokens. */
  39   public String sval;
  40
  41   /* The numeric value associated with number tokens. */
  42   public double nval;
  43
  44   /* Indicates whether end-of-line is recognized as a token. */
  45   private boolean eolSignificant = false;
  46
  47   /* Indicates whether word tokens are automatically made lower case. */
  48   private boolean lowerCase = false;
  49
  50   /* Indicates whether C++ style comments are recognized and skipped. */
  51   private boolean slashSlash = false;
  52
  53   /* Indicates whether C style comments are recognized and skipped. */
  54   private boolean slashStar = false;
  55
  56   /* Attribute tables of each byte from 0x00 to 0xFF. */
  57   private boolean[] whitespace;
  58   private boolean[] alphabetic;
  59   private boolean[] numeric;
  60   private boolean[] quote;
  61   private boolean[] comment;
  62
  63   /* The Reader associated with this class. */
  64   private PushbackReader in;
  65
  66   /* Indicates if a token has been pushed back. */
  67   private boolean pushedBack = false;
  68
  69   /* Contains the current line number of the reader. */
  70   private int lineNumber = 1;
  71
  72   // Deprecated in JDK 1.1.
  73   public StreamTokenizer(InputStream is)
  74   {
  75     this(new InputStreamReader(is));
  76   }
  77
  78   public StreamTokenizer(Reader r)
  79   {
  80     in = new PushbackReader(r);
  81
  82     whitespace = new boolean[256];
  83     alphabetic = new boolean[256];
  84     numeric = new boolean[256];
  85     quote = new boolean[256];
  86     comment = new boolean[256];
  87     for (int i = 0; i < 256; i++)
  88       resetChar(i);
  89
  90     whitespaceChars(0x00, 0x20);
  91     wordChars('A', 'Z');
  92     wordChars('a', 'z');
  93     wordChars(0xA0, 0xFF);
  94     commentChar('/');
  95     quoteChar('\'');
  96     quoteChar('"');
  97     parseNumbers();
  98   }
  99
 100   public void commentChar(int ch)
 101   {
 102     if (ch >= 0 && ch <= 255)
 103       comment[ch] = true;
 104   }
 105
 106   public void eolIsSignificant(boolean flag)
 107   {
 108     eolSignificant = flag;
 109   }
 110
 111   public int lineno()
 112   {
 113     return lineNumber;
 114   }
 115
 116   public void lowerCaseMode(boolean flag)
 117   {
 118     lowerCase = flag;
 119   }
 120
 121   private boolean isWhitespace(int ch)
 122   {
 123     if (ch >= 0 && ch <= 255)
 124       return whitespace[ch];
 125
 126     return false;
 127   }
 128
 129   private boolean isAlphabetic(int ch)
 130   {
 131     if (ch >= 0 && ch <= 255)
 132       return alphabetic[ch];
 133     else if (ch > 255)
 134       return true;
 135
 136     return false;
 137   }
 138
 139   private boolean isNumeric(int ch)
 140   {
 141     if (ch >= 0 && ch <= 255)
 142       return numeric[ch];
 143
 144     return false;
 145   }
 146
 147   private boolean isQuote(int ch)
 148   {
 149     if (ch >= 0 && ch <= 255)
 150       return quote[ch];
 151
 152     return false;
 153   }
 154
 155   private boolean isComment(int ch)
 156   {
 157     if (ch >= 0 && ch <= 255)
 158       return comment[ch];
 159
 160     return false;
 161   }
 162
 163   public int nextToken() throws IOException
 164   {
 165     if (pushedBack)
 166       {
 167         pushedBack = false;
 168         return ttype;
 169       }
 170
 171     sval = null;
 172     int ch;
 173
 174     // Skip whitespace.  Deal with EOL along the way.
 175     while (isWhitespace(ch = in.read()))
 176       if (ch == '\n' || ch == '\r')
 177         {
 178           lineNumber++;
 179
 180           // Throw away \n if in combination with \r.
 181           if (ch == '\r' && (ch = in.read()) != '\n')
 182             in.unread(ch);
 183           if (eolSignificant)
 184             return (ttype = TT_EOL);
 185         }
 186
 187     if (ch == TT_EOF)
 188       ttype = TT_EOF;
 189     else if (isNumeric(ch))
 190       {
 191         if (ch == '-')
 192           {
 193             // Read ahead to see if this is an ordinary '-' rather than numeric.
 194             ch = in.read();
 195             in.unread(ch);
 196             if (isNumeric(ch) && ch != '-')
 197               ch = '-';
 198             else
 199               return (ttype = '-');
 200           }
 201
 202         StringBuffer tokbuf = new StringBuffer();
 203         tokbuf.append((char) ch);
 204
 205         int decCount = 0;
 206         while (isNumeric(ch = in.read()) && ch != '-')
 207           if (ch == '.' && decCount++ > 0)
 208             break;
 209           else
 210             tokbuf.append((char) ch);
 211
 212         in.unread(ch);
 213         ttype = TT_NUMBER;
 214         nval = Double.valueOf(tokbuf.toString()).doubleValue();
 215       }
 216     else if (isAlphabetic(ch))
 217       {
 218         StringBuffer tokbuf = new StringBuffer();
 219         tokbuf.append((char) ch);
 220         while (isAlphabetic(ch = in.read()) || isNumeric(ch))
 221           tokbuf.append((char) ch);
 222         in.unread(ch);
 223         ttype = TT_WORD;
 224         sval = tokbuf.toString();
 225         if (lowerCase)
 226           sval.toLowerCase();
 227       }
 228     else if (isComment(ch))
 229       {
 230         while ((ch = in.read()) != '\n' && ch != '\r' && ch != TT_EOF)
 231           ;
 232         in.unread(ch);
 233         return nextToken();     // Recursive, but not too deep in normal cases.
 234       }
 235     else if (isQuote(ch))
 236       {
 237         ttype = ch;
 238         StringBuffer tokbuf = new StringBuffer();
 239         while ((ch = in.read()) != ttype && ch != '\n' && ch != '\r' &&
 240                ch != TT_EOF)
 241           {
 242             if (ch == '\\')
 243               switch (ch = in.read())
 244                 {
 245                   case 'a':     ch = 0x7;
 246                     break;
 247                   case 'b':     ch = '\b';
 248                     break;
 249                   case 'f':     ch = 0xC;
 250                     break;
 251                   case 'n':     ch = '\n';
 252                     break;
 253                   case 'r':     ch = '\r';
 254                     break;
 255                   case 't':     ch = '\t';
 256                     break;
 257                   case 'v':     ch = 0xB;
 258                     break;
 259                   case '\"':
 260                   case '\'':
 261                   case '\\':
 262                     break;
 263                   default:
 264                     int ch1, nextch;
 265                     if ((nextch = ch1 = ch) >= '0' && ch <= '7')
 266                       {
 267                         ch -= '0';
 268                         if ((nextch = in.read()) >= '0' && nextch <= '7')
 269                           {
 270                             ch = ch * 8 + nextch - '0';
 271                             if ((nextch = in.read()) >= '0' && nextch <= '7' &&
 272                                 ch1 >= '0' && ch1 <= '3')
 273                               {
 274                                 ch = ch * 8 + nextch - '0';
 275                                 nextch = in.read();
 276                               }
 277                           }
 278                       }
 279
 280                     in.unread(nextch);
 281                 }
 282
 283             tokbuf.append((char) ch);
 284           }
 285
 286         // Throw away matching quote char.
 287         if (ch != ttype)
 288           in.unread(ch);
 289
 290         sval = tokbuf.toString();
 291       }
 292     else
 293       {
 294         if (ch == '/')
 295           if ((ch = in.read()) == '/' && slashSlash)
 296             {
 297               while ((ch = in.read()) != '\n' && ch != '\r' && ch != TT_EOF)
 298                 ;
 299               in.unread(ch);
 300               return nextToken(); // Recursive, but not too deep in normal cases
 301             }
 302           else if (ch == '*' && slashStar)
 303             {
 304               while (true)
 305                 {
 306                   ch = in.read();
 307                   if (ch == '*')
 308                     if ((ch = in.read()) == '/')
 309                       break;
 310                     else
 311                       in.unread(ch);
 312                   else if (ch == '\n' || ch == '\r')
 313                     {
 314                       lineNumber++;
 315                       if (ch == '\r' && (ch = in.read()) != '\n')
 316                         in.unread(ch);
 317                     }
 318                   else if (ch == TT_EOF)
 319                     {
 320                       in.unread(ch);
 321                       break;
 322                     }
 323                 }
 324               return nextToken(); // Recursive, but not too deep in normal cases
 325             }
 326           else
 327             {
 328               in.unread(ch);
 329               ch = '/';
 330             }
 331
 332         ttype = ch;
 333       }
 334
 335     return ttype;
 336   }
 337
 338   private void resetChar(int ch)
 339   {
 340     whitespace[ch] = alphabetic[ch] = numeric[ch] = quote[ch] = comment[ch] =
 341       false;
 342   }
 343
 344   public void ordinaryChar(int ch)
 345   {
 346     if (ch >= 0 && ch <= 255)
 347       resetChar(ch);
 348   }
 349
 350   public void ordinaryChars(int low, int hi)
 351   {
 352     if (low < 0)
 353       low = 0;
 354     if (hi > 255)
 355       hi = 255;
 356     for (int i = low; i <= hi; i++)
 357       resetChar(i);
 358   }
 359
 360   public void parseNumbers()
 361   {
 362     for (int i = 0; i <= 9; i++)
 363       numeric['0' + i] = true;
 364
 365     numeric['.'] = true;
 366     numeric['-'] = true;
 367   }
 368
 369   public void pushBack()
 370   {
 371     // pushBack may cause the lineno method to return an incorrect value
 372     // if lineno is called before the next call to nextToken.
 373     pushedBack = true;
 374   }
 375
 376   public void quoteChar(int ch)
 377   {
 378     if (ch >= 0 && ch <= 255)
 379       quote[ch] = true;
 380   }
 381
 382   public void resetSyntax()
 383   {
 384     ordinaryChars(0x00, 0xFF);
 385   }
 386
 387   public void slashSlashComments(boolean flag)
 388   {
 389     slashSlash = flag;
 390   }
 391
 392   public void slashStarComments(boolean flag)
 393   {
 394     slashStar = flag;
 395   }
 396
 397   public String toString()
 398   {
 399     String tempstr;
 400     if (ttype == TT_EOF)
 401       tempstr = "EOF";
 402     else if (ttype == TT_EOL)
 403       tempstr = "EOL";
 404     else if (ttype == TT_WORD)
 405       tempstr = sval;
 406     else if (ttype == TT_NUMBER)
 407       tempstr = "n=" + Double.toString(nval);
 408     else // must be an ordinary char.
 409       tempstr = "\'" + (new Character((char) ttype)).toString() + "\'";
 410
 411     return "Token[" + tempstr + "], line " + Integer.toString(lineno());
 412   }
 413
 414   public void whitespaceChars(int low, int hi)
 415   {
 416     if (low < 0)
 417       low = 0;
 418     if (hi > 255)
 419       hi = 255;
 420     for (int i = low; i <= hi; i++)
 421       whitespace[i] = true;
 422   }
 423
 424   public void wordChars(int low, int hi)
 425   {
 426     if (low < 0)
 427       low = 0;
 428     if (hi > 255)
 429       hi = 255;
 430     for (int i = low; i <= hi; i++)
 431       alphabetic[i] = true;
 432   }
 433 }