This is the mail archive of the
gcc-patches@gcc.gnu.org
mailing list for the GCC project.
Traditional lexing
- From: Neil Booth <neil at daikokuya dot demon dot co dot uk>
- To: gcc-patches at gcc dot gnu dot org
- Date: Fri, 17 May 2002 00:03:48 +0100
- Subject: Traditional lexing
This patch implements the main lexing loop for traditional
preprocessing. It lexes logical lines, skipping comments
(outside quotes) and splicing escaped newlines. Traditional
macro expansion will be done from this loop too, once I've
implemented it.
If we're compiling, then the result is passed on to the ISO
lexer for ISO tokenization (possibly with minor differences
in such things as numbers); if we're preprocessing only
it would be dumped out as-is.
In implementing traditional preprocessing, I don't want to
have it kill the ISO preprocessor performance-wise. In
particular, any solution that involved having something
like
if (traditional)
lex_traditionally ();
else
lex_iso ();
in _cpp_lex_token() or _cpp_lex_direct() is unacceptable to me.
So instead I fool the ISO lexer into thinking it's preprocessing
the original file, by a method I've called buffer overlays,
which adds truly negligible overhead to the main ISO preprocessor.
When the traditional preprocessor has finished scanning a logical
line to a scratch buffer, it NUL-terminates it and temporarily
replaces the true file buffer's start and end pointers with those
of the (eventually macro-replaced) logical line in the scratch
buffer.
This means that traditional lexing (outside of directives) can be
completely handled by the normal ISO lexer, with line refills
handled in the little-taken '\0' case that normally handles EOF.
I've adjusted that code to detect if we're preprocessing
traditionally, and in that case it asks the traditional code
to refill the scratch buffer with the next logical line, update
the buffer's pointers again, and then continues as if it had
never seen a NUL.
Thus traditional preprocessing is handled within the existing
token-lexing framework (in the compiling case). The only problem
remaining is how to get this started; the above description
assumes the process is already underway and that there is something
for the first call to cpp_get_token() to lex. Obviously having
an if statment at the beginning of _cpp_lex_token() would defeat
the whole point. But it is easily solved by the cpplib
initialization routines doing an initial buffer overlay of the
empty string, ensuring we go to the NUL handler for a logical line
refill 8-)
The patch below implements this. It doesn't handle traditional
cpp output as a special case in cppmain.c yet; it just goes through
the ISO lexer like the compiler front end would. This means that
for the moment unmatched quotes etc. give errors. But it's good
enough to test that the other features of traditional preprocessing
(like /**/ pasting) and line splicing already work.
Built-ins and command-line macros are turned off for the moment,
and directives are ignored (the ISO preprocessor probably
erroneously picks them up).
cc1 does traditional preprocessing with the -traditional-cpp
switch. If anyone prefers -traditional for any reason let me know.
I'll apply this when a bootstrap finishes.
Neil.
* Makefile.in: Update for cpptrad.c.
* cpphash.h (struct cpp_buffer): New members for buffer
overlays.
(struct cpp_reader): New members for traditional output.
(_cpp_read_logical_line, _cpp_overlay_buffer): New.
* cppinit.c (cpp_create_reader): Set trad_line.
(cpp_destroy): Free trad_out_base if used.
(cpp_read_main_file): Overlay an empty buffer if traditional.
(cpp_finish_options): Don't do builtins.
(COMMAND_LINE_OPTIONS): Add -traditional-cpp.
(cpp_handle_option): Handle it.
* cpplex.c (handle_nul): New.
(_cpp_lex_direct): Use handle_nul.
* cpplib.h (struct cpp_options): New traditional option.
* cpptrad.c: New file.
============================================================
Index: gcc/Makefile.in
--- gcc/Makefile.in 16 May 2002 17:42:20 -0000 1.874
+++ gcc/Makefile.in 16 May 2002 23:00:48 -0000
@@ -1993,7 +1993,7 @@ PREPROCESSOR_DEFINES = \
-DCROSS_INCLUDE_DIR=\"$(gcc_tooldir)/sys-include\" \
-DTOOL_INCLUDE_DIR=\"$(gcc_tooldir)/include\"
-LIBCPP_OBJS = cpplib.o cpplex.o cppmacro.o cppexp.o cppfiles.o \
+LIBCPP_OBJS = cpplib.o cpplex.o cppmacro.o cppexp.o cppfiles.o cpptrad.o \
cpphash.o cpperror.o cppinit.o cppdefault.o cppmain.o \
hashtable.o line-map.o mkdeps.o prefix.o version.o mbchar.o
@@ -2015,6 +2015,7 @@ cpplex.o: cpplex.c $(CONFIG_H) $(LIB
cppmacro.o: cppmacro.c $(CONFIG_H) $(LIBCPP_DEPS)
cpplib.o: cpplib.c $(CONFIG_H) $(LIBCPP_DEPS)
cpphash.o: cpphash.c $(CONFIG_H) $(LIBCPP_DEPS)
+cpptrad.o: cpptrad.c $(CONFIG_H) $(LIBCPP_DEPS)
cppfiles.o: cppfiles.c $(CONFIG_H) $(LIBCPP_DEPS) $(SPLAY_TREE_H) mkdeps.h
cppinit.o: cppinit.c $(CONFIG_H) $(LIBCPP_DEPS) cppdefault.h \
mkdeps.h prefix.h version.h
============================================================
Index: gcc/cpphash.h
--- gcc/cpphash.h 16 May 2002 05:53:23 -0000 1.149
+++ gcc/cpphash.h 16 May 2002 23:00:50 -0000
@@ -245,6 +245,9 @@ struct cpp_buffer
/* The directory of the this buffer's file. Its NAME member is not
allocated, so we don't need to worry about freeing it. */
struct search_path dir;
+
+ /* Used for buffer overlays by cpptrad.c. */
+ const uchar *saved_cur, *saved_rlimit, *saved_line_base;
};
/* A cpp_reader encapsulates the "state" of a pre-processor run.
@@ -355,6 +358,11 @@ struct cpp_reader
/* Whether cpplib owns the hashtable. */
unsigned char our_hashtable;
+
+ /* Traditional preprocessing output buffer. */
+ uchar *trad_out_base, *trad_out_limit;
+ uchar *trad_out_cur;
+ unsigned int trad_line;
};
/* Character classes. Based on the more primitive macros in safe-ctype.h.
@@ -446,6 +454,11 @@ extern void _cpp_do_file_change PARAMS (
const char *,
unsigned int, unsigned int));
extern void _cpp_pop_buffer PARAMS ((cpp_reader *));
+
+/* In cpptrad.c. */
+extern bool _cpp_read_logical_line_trad PARAMS ((cpp_reader *));
+extern void _cpp_overlay_buffer PARAMS ((cpp_reader *pfile, const uchar *,
+ size_t));
/* Utility routines and macros. */
#define DSC(str) (const uchar *)str, sizeof str - 1
============================================================
Index: gcc/cppinit.c
--- gcc/cppinit.c 16 May 2002 19:02:59 -0000 1.224
+++ gcc/cppinit.c 16 May 2002 23:00:54 -0000
@@ -514,7 +514,7 @@ cpp_create_reader (lang)
/* Initialise the line map. Start at logical line 1, so we can use
a line number of zero for special states. */
init_line_maps (&pfile->line_maps);
- pfile->line = 1;
+ pfile->trad_line = pfile->line = 1;
/* Initialize lexer state. */
pfile->state.save_comments = ! CPP_OPTION (pfile, discard_comments);
@@ -569,6 +569,9 @@ cpp_destroy (pfile)
while (CPP_BUFFER (pfile) != NULL)
_cpp_pop_buffer (pfile);
+ if (pfile->trad_out_base)
+ free (pfile->trad_out_base);
+
if (pfile->macro_buffer)
{
free ((PTR) pfile->macro_buffer);
@@ -954,6 +957,9 @@ cpp_read_main_file (pfile, fname, table)
of the front ends. */
if (CPP_OPTION (pfile, preprocessed))
read_original_filename (pfile);
+ /* Overlay an empty buffer to seed traditional preprocessing. */
+ else if (CPP_OPTION (pfile, traditional))
+ _cpp_overlay_buffer (pfile, U"", 0);
return pfile->map->to_file;
}
@@ -1002,10 +1008,12 @@ cpp_finish_options (pfile)
struct pending_option *p;
_cpp_do_file_change (pfile, LC_RENAME, _("<built-in>"), 1, 0);
- init_builtins (pfile);
+ if (!CPP_OPTION (pfile, traditional) /* REMOVEME */)
+ init_builtins (pfile);
_cpp_do_file_change (pfile, LC_RENAME, _("<command line>"), 1, 0);
- for (p = CPP_OPTION (pfile, pending)->directive_head; p; p = p->next)
- (*p->handler) (pfile, p->arg);
+ if (!CPP_OPTION (pfile, traditional) /* REMOVEME */)
+ for (p = CPP_OPTION (pfile, pending)->directive_head; p; p = p->next)
+ (*p->handler) (pfile, p->arg);
/* Scan -imacros files after -D, -U, but before -include.
pfile->next_include_file is NULL, so _cpp_pop_buffer does not
@@ -1201,6 +1209,7 @@ new_pending_directive (pend, text, handl
DEF_OPT("std=iso9899:199409", 0, OPT_std_iso9899_199409) \
DEF_OPT("std=iso9899:1999", 0, OPT_std_iso9899_1999) \
DEF_OPT("std=iso9899:199x", 0, OPT_std_iso9899_199x) \
+ DEF_OPT("traditional-cpp", 0, OPT_traditional_cpp) \
DEF_OPT("trigraphs", 0, OPT_trigraphs) \
DEF_OPT("v", 0, OPT_v) \
DEF_OPT("version", 0, OPT_version) \
@@ -1453,6 +1462,9 @@ cpp_handle_option (pfile, argc, argv, ig
break;
case OPT_remap:
CPP_OPTION (pfile, remap) = 1;
+ break;
+ case OPT_traditional_cpp:
+ CPP_OPTION (pfile, traditional) = 1;
break;
case OPT_iprefix:
CPP_OPTION (pfile, include_prefix) = arg;
============================================================
Index: gcc/cpplex.c
--- gcc/cpplex.c 11 May 2002 21:02:33 -0000 1.206
+++ gcc/cpplex.c 16 May 2002 23:00:59 -0000
@@ -78,6 +78,7 @@ static void parse_string PARAMS ((cpp_re
static bool trigraph_p PARAMS ((cpp_reader *));
static void save_comment PARAMS ((cpp_reader *, cpp_token *, const uchar *,
cppchar_t));
+static bool handle_nul PARAMS ((cpp_reader *));
static int name_p PARAMS ((cpp_reader *, const cpp_string *));
static int maybe_read_ucs PARAMS ((cpp_reader *, const unsigned char **,
const unsigned char *, cppchar_t *));
@@ -877,6 +878,48 @@ _cpp_lex_token (pfile)
return result;
}
+/* A NUL terminates the current buffer. For ISO preprocessing this is
+ EOF, but for traditional preprocessing it indicates we need a line
+ refill. Returns TRUE to continue preprocessing a new buffer, FALSE
+ to return a CPP_EOF to the caller. */
+static bool
+handle_nul (pfile)
+ cpp_reader *pfile;
+{
+ cpp_buffer *buffer = pfile->buffer;
+ bool more = true;
+
+ buffer->saved_flags = BOL;
+ if (CPP_OPTION (pfile, traditional))
+ more = _cpp_read_logical_line_trad (pfile);
+ else
+ {
+ /* Stop parsing arguments with a CPP_EOF. We we finally come
+ back here, do the work of popping the buffer. */
+ if (!pfile->state.parsing_args)
+ {
+ if (buffer->cur != buffer->line_base)
+ {
+ /* Non-empty files should end in a newline. Don't warn
+ for command line and _Pragma buffers. */
+ if (!buffer->from_stage3)
+ cpp_error (pfile, DL_PEDWARN, "no newline at end of file");
+ handle_newline (pfile);
+ }
+
+ /* Similarly, finish an in-progress directive with CPP_EOF
+ before popping the buffer. */
+ if (!pfile->state.in_directive && buffer->prev)
+ {
+ more = !buffer->return_at_eof;
+ _cpp_pop_buffer (pfile);
+ }
+ }
+ }
+
+ return more;
+}
+
#define IF_NEXT_IS(CHAR, THEN_TYPE, ELSE_TYPE) \
do { \
if (get_effective_char (pfile) == CHAR) \
@@ -927,30 +970,10 @@ _cpp_lex_direct (pfile)
if (skip_whitespace (pfile, c))
goto skipped_white;
- /* EOF. */
+ /* End of buffer. */
buffer->cur--;
- buffer->saved_flags = BOL;
- if (!pfile->state.parsing_args)
- {
- if (buffer->cur != buffer->line_base)
- {
- /* Non-empty files should end in a newline. Don't warn
- for command line and _Pragma buffers. */
- if (!buffer->from_stage3)
- cpp_error (pfile, DL_PEDWARN, "no newline at end of file");
- handle_newline (pfile);
- }
-
- /* Don't pop the last buffer. */
- if (!pfile->state.in_directive && buffer->prev)
- {
- unsigned char stop = buffer->return_at_eof;
-
- _cpp_pop_buffer (pfile);
- if (!stop)
- goto fresh_line;
- }
- }
+ if (handle_nul (pfile))
+ goto fresh_line;
result->type = CPP_EOF;
break;
============================================================
Index: gcc/cpplib.h
--- gcc/cpplib.h 16 May 2002 19:03:00 -0000 1.217
+++ gcc/cpplib.h 16 May 2002 23:01:01 -0000
@@ -386,6 +386,9 @@ struct cpp_options
parsing; drivers might want to continue printing help. */
unsigned char help_only;
+ /* True for traditional preprocessing. */
+ unsigned char traditional;
+
/* Target-specific features set by the front end or client. */
/* Precision for target CPP arithmetic, target characters, target
============================================================
Index: gcc/cpptrad.c
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ gcc/cpptrad.c 16 May 2002 23:01:01 -0000
@@ -0,0 +1,315 @@
+/* CPP Library - traditional lexical analysis and macro expansion.
+ Copyright (C) 2002 Free Software Foundation, Inc.
+ Contributed by Neil Booth, May 2002
+
+This program is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 2, or (at your option) any
+later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
+
+#include "config.h"
+#include "system.h"
+#include "cpplib.h"
+#include "cpphash.h"
+
+/* Lexing TODO: Handle -Wcomment, -C, maybe -CC, and space in escaped
+ newlines. Stop cpplex.c from recognizing comments, trigraphs and
+ directives during its lexing pass. */
+
+static const uchar *handle_newline PARAMS ((cpp_reader *, const uchar *));
+static const uchar *skip_escaped_newlines PARAMS ((cpp_reader *,
+ const uchar *));
+static cpp_hashnode *lex_identifier PARAMS ((cpp_reader *, const uchar *));
+static const uchar *skip_comment PARAMS ((cpp_reader *, const uchar *));
+static void scan_out_logical_line PARAMS ((cpp_reader *pfile));
+static void check_output_buffer PARAMS ((cpp_reader *, size_t));
+static void restore_buff PARAMS ((cpp_reader *));
+
+/* Ensures we have N bytes' space in the output buffer, and
+ reallocates it if not. */
+static void
+check_output_buffer (pfile, n)
+ cpp_reader *pfile;
+ size_t n;
+{
+ if (n > (size_t) (pfile->trad_out_limit - pfile->trad_out_cur))
+ {
+ size_t size = pfile->trad_out_cur - pfile->trad_out_base;
+ size_t new_size = (size + n) * 3 / 2;
+
+ pfile->trad_out_base
+ = (uchar *) xrealloc (pfile->trad_out_base, new_size);
+ pfile->trad_out_limit = pfile->trad_out_base + new_size;
+ pfile->trad_out_cur = pfile->trad_out_base + size;
+ }
+}
+
+/* To be called whenever a newline character is encountered in the
+ input file, at CUR. Handles DOS, MAC and Unix ends of line, and
+ returns the character after the newline sequence. */
+static const uchar *
+handle_newline (pfile, cur)
+ cpp_reader *pfile;
+ const uchar *cur;
+{
+ pfile->line++;
+ if (cur[0] + cur[1] == '\r' + '\n')
+ cur++;
+ return cur + 1;
+}
+
+/* CUR points to any character in the buffer, not necessarily a
+ backslash. Advances CUR until all escaped newlines are skipped,
+ and returns the new position. */
+static const uchar *
+skip_escaped_newlines (pfile, cur)
+ cpp_reader *pfile;
+ const uchar *cur;
+{
+ while (*cur == '\\' && is_vspace (cur[1]))
+ cur = handle_newline (pfile, cur + 1);
+
+ return cur;
+}
+
+/* CUR points to the character after the asterisk introducing a
+ comment. Returns the position after the comment. */
+static const uchar *
+skip_comment (pfile, cur)
+ cpp_reader *pfile;
+ const uchar *cur;
+{
+ unsigned int from_line = pfile->line;
+
+ for (;;)
+ {
+ unsigned int c = *cur++;
+ if (c == '*')
+ {
+ cur = skip_escaped_newlines (pfile, cur);
+ if (*cur == '/')
+ {
+ cur++;
+ break;
+ }
+ }
+ else if (is_vspace (c))
+ cur = handle_newline (pfile, cur - 1);
+ else if (c == '\0' && cur - 1 == pfile->buffer->rlimit)
+ {
+ cur--;
+ cpp_error_with_line (pfile, DL_ERROR, from_line, 0,
+ "unterminated comment");
+ break;
+ }
+ }
+
+ return cur;
+}
+
+/* Lexes and outputs an identifier starting at CUR, which is assumed
+ to point to a valid first character of an identifier. Returns
+ the hashnode, and updates trad_out_cur. */
+static cpp_hashnode *
+lex_identifier (pfile, cur)
+ cpp_reader *pfile;
+ const uchar *cur;
+{
+ size_t len;
+ uchar *out = pfile->trad_out_cur;
+
+ do
+ {
+ do
+ *out++ = *cur++;
+ while (ISIDNUM (*cur));
+ cur = skip_escaped_newlines (pfile, cur);
+ }
+ while (ISIDNUM (*cur));
+
+ pfile->buffer->cur = cur;
+ len = out - pfile->trad_out_cur;
+ pfile->trad_out_cur = out;
+ return (cpp_hashnode *) ht_lookup (pfile->hash_table, pfile->trad_out_cur,
+ len, HT_ALLOC);
+}
+
+/* Overlays the true file buffer temporarily with text of length LEN
+ starting at START. The true buffer is restored upon calling
+ restore_buff(). */
+void
+_cpp_overlay_buffer (pfile, start, len)
+ cpp_reader *pfile;
+ const uchar *start;
+ size_t len;
+{
+ cpp_buffer *buffer = pfile->buffer;
+
+ buffer->saved_cur = buffer->cur;
+ buffer->saved_rlimit = buffer->rlimit;
+ buffer->saved_line_base = buffer->line_base;
+
+ buffer->cur = start;
+ buffer->line_base = start;
+ buffer->rlimit = start + len;
+}
+
+/* Restores a buffer overlaid by _cpp_overlay_buffer(). */
+static void
+restore_buff (pfile)
+ cpp_reader *pfile;
+{
+ cpp_buffer *buffer = pfile->buffer;
+
+ buffer->cur = buffer->saved_cur;
+ buffer->rlimit = buffer->saved_rlimit;
+ buffer->line_base = buffer->saved_line_base;
+}
+
+/* Reads a logical line into the output buffer. Returns TRUE if there
+ is more text left in the buffer. */
+bool
+_cpp_read_logical_line_trad (pfile)
+ cpp_reader *pfile;
+{
+ cpp_buffer *buffer;
+ unsigned int first_line;
+
+ restore_buff (pfile);
+
+ first_line = pfile->line = pfile->trad_line;
+
+ buffer = pfile->buffer;
+ if (buffer->cur == buffer->rlimit)
+ {
+ bool stop = true;
+
+ /* Don't pop the last buffer. */
+ if (buffer->prev)
+ {
+ stop = buffer->return_at_eof;
+ _cpp_pop_buffer (pfile);
+ }
+
+ if (stop)
+ return false;
+ }
+
+ pfile->trad_out_cur = pfile->trad_out_base;
+ scan_out_logical_line (pfile);
+ pfile->trad_line = pfile->line;
+ pfile->line = first_line;
+ _cpp_overlay_buffer (pfile, pfile->trad_out_base,
+ pfile->trad_out_cur - pfile->trad_out_base);
+ return true;
+}
+
+/* Copies the next logical line in the current buffer to the output
+ buffer. The output is guaranteed to terminate with a NUL
+ character. */
+static void
+scan_out_logical_line (pfile)
+ cpp_reader *pfile;
+{
+ cpp_buffer *buffer = pfile->buffer;
+ const uchar *cur = buffer->cur;
+ unsigned int c, quote = 0;
+ uchar *out;
+
+ check_output_buffer (pfile, buffer->rlimit - cur);
+ out = pfile->trad_out_cur;
+
+ for (;;)
+ {
+ c = *cur++;
+ *out++ = c;
+
+ /* There are only a few entities we need to catch: comments,
+ identifiers, newlines, escaped newlines, # and '\0'. */
+ switch (c)
+ {
+ case '\0':
+ if (cur - 1 != buffer->rlimit)
+ break;
+ cur--;
+ if (!buffer->from_stage3)
+ cpp_error (pfile, DL_PEDWARN, "no newline at end of file");
+ pfile->line++;
+ if (0)
+ {
+ case '\r': case '\n':
+ cur = handle_newline (pfile, cur - 1);
+ }
+ out[-1] = '\n';
+ out[0] = '\0';
+ buffer->cur = cur;
+ pfile->trad_out_cur = out;
+ return;
+
+ case '"':
+ case '\'':
+ if (c == quote)
+ quote = 0;
+ else if (!quote)
+ quote = c;
+ break;
+
+ case '\\':
+ if (is_vspace (*cur))
+ out--, cur = skip_escaped_newlines (pfile, cur - 1);
+ else
+ {
+ /* Skip escaped quotes here, it's easier than above, but
+ take care to first skip escaped newlines. */
+ cur = skip_escaped_newlines (pfile, cur);
+ if (*cur == '\\' || *cur == '"' || *cur == '\'')
+ *out++ = *cur++;
+ }
+ break;
+
+ case '/':
+ /* Traditional CPP does not recognize comments within
+ literals. */
+ if (!quote)
+ {
+ cur = skip_escaped_newlines (pfile, cur);
+ if (*cur == '*')
+ out--, cur = skip_comment (pfile, cur + 1);
+ }
+ break;
+
+ case '_':
+ case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
+ case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
+ case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
+ case 's': case 't': case 'u': case 'v': case 'w': case 'x':
+ case 'y': case 'z':
+ case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
+ case 'G': case 'H': case 'I': case 'J': case 'K':
+ case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
+ case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
+ case 'Y': case 'Z':
+ {
+ cpp_hashnode *node;
+
+ pfile->trad_out_cur = --out;
+ node = lex_identifier (pfile, cur - 1);
+ out = pfile->trad_out_cur;
+ cur = buffer->cur;
+ }
+ break;
+
+ default:
+ break;
+ }
+ }
+}