Re: gcc macro collection

[Yunfeng ZHANG <zyf dot zeroos at gmail dot com>]

> For the purpose of your plugin, would it have been enough to have an API
> that, for each preprocessed token, would tell you if that token is part
> of a macro replacement-list, is a macro definition token, or is not
> related to a macro at all? For a macro related token, the API would of
> course give you the location of the place where the macro was first defined.

What you want to know?
About line_map, I still recommends my cpp_token::file_offset field since it's
simpler than current line_map and source_location combination. When
file_offset is less than zero, it can be used to indicate the token is macro
replacement token.

> Why using the variable plugin_tok here? Why not using the tok variable
> directly?

If deliver tok to invoke_plugin_callbacks directly, you will receive a warn
just like `try to pass a const pointer to a function whose parameters are
non-const'.


-------------------------
I resent my patch following the advices and added a missed callback
`directive_token' which is also important to collect macro expansion,
(by the way original patch is published from a mistake patch stack).

Here is a brief description about how using the callbacks and new event.

Consider the case
   ----------- a.c -----------
   #define FOO \
       = 2;
   int x FOO;
token FOO is substituted to `=' and `2', so
FOO is called as EXPANDED_TOKEN.
`=' and `2' are called as MACRO_TOKEN.

To use my interface to collect the tokens, you should use gcc4.5 plugin
architecture, later default directory is my attachment, default file is
symdb.txt and symdb.c.
1) Open Makefile and symdb.c:<plugin> fold to learn how to write your plugin.
2) Your adapter class should implement not only new four callbacks but also
an old file_change callback (symdb_token_init()), file_change callback is used
to set up a file-dependent stack just like `gcc -MP' option family. Then
tokens are catched from PLUGIN_CPP_TOKEN one-by-one, and my callbacks will
notice you which are COMMON_TOKEN, which are MACRO_TOKEN (cpp_callbacks
fold).
3) test/testplan.txt includes my testcases on my patch, generally, your plugin
must pass the cases listed in them.
4) In fact, EXPANDED_TOKEN can't be got directly from PLUGIN_CPP_TOKEN,
there're two kinds of EXPANDED_TOKEN, one is your .c/.h which are called
  1-level EXPANDED_TOKEN, uses cb_directive_token to collect them;
  2+-level EXPANDED_TOKEN are also the result of the former macro expansion,
  using cb_intern_expand.

Patch 1 of 2 on libcpp/
-------------------------
Index: gcc/libcpp/include/cpplib.h
===================================================================
--- gcc.orig/libcpp/include/cpplib.h
+++ gcc/libcpp/include/cpplib.h
@@ -495,6 +495,25 @@ struct cpp_callbacks
      be expanded.  */
   cpp_hashnode * (*macro_to_expand) (cpp_reader *, const cpp_token *);

+  /* macro_{start/end}_expand are called when gcc starts to expand macro, note
+   * if A macro includes B macro, the pair is called multiple times. */
+  void (*macro_start_expand) (cpp_reader *, const cpp_token *,
+                 const cpp_hashnode *);
+  void (*macro_end_expand) (cpp_reader *, const cpp_hashnode *);
+  /* Called when a function-like macro stops collecting macro arguments,
+   * cancel = true when gcc prefetches two tokens `x' and `=' and realizes
+   * macro expansion should be canceled. */
+  void (*macro_end_arg) (cpp_reader *, bool cancel);
+  /* Later two callbacks are used to catch those tokens which are
+   * expanded/replaced, normally they are invisible to user.
+   * A macro (in .c/.h files) includes B macro,
+   *   *) here `A(...)' are called 1-level expanded tokens. Using
+   *   directive_token to catch them.
+   *   *) 2+-level expanded tokens should use macro_intern_expand.
+   * By the way, directive_token can catch all tokens in .c. */
+  void (*directive_token) (cpp_reader *, const cpp_token*);
+  void (*macro_intern_expand) (cpp_reader *, void*, int, bool);
+
   /* Called to emit a diagnostic.  This callback receives the
      translated message.  */
   bool (*error) (cpp_reader *, int, source_location, unsigned int,
Index: gcc/libcpp/macro.c
===================================================================
--- gcc.orig/libcpp/macro.c
+++ gcc/libcpp/macro.c
@@ -848,10 +848,13 @@ enter_macro_context (cpp_reader *pfile,
       if (macro->fun_like)
 	{
 	  _cpp_buff *buff;
+	  void *expanded_beg = NULL, *expanded_end;

 	  pfile->state.prevent_expansion++;
 	  pfile->keep_tokens++;
 	  pfile->state.parsing_args = 1;
+	  if (pfile->cb.macro_intern_expand)
+		expanded_beg = (void *) FIRST (pfile->context).token;
 	  buff = funlike_invocation_p (pfile, node, &pragma_buff);
 	  pfile->state.parsing_args = 0;
 	  pfile->keep_tokens--;
@@ -867,9 +870,25 @@ enter_macro_context (cpp_reader *pfile,
 	      if (pragma_buff)
 		_cpp_release_buff (pfile, pragma_buff);

+	      if (pfile->cb.macro_end_arg)
+		pfile->cb.macro_end_arg (pfile, true);
 	      return 0;
 	    }
+	  if (pfile->cb.macro_intern_expand)
+	    {
+	      cpp_context *context = pfile->context;
+	      expanded_end = (void *) FIRST (context).token;
+	      pfile->cb.macro_intern_expand (pfile, expanded_beg,
+				 context->direct_p ?
+				 (cpp_token *) expanded_end -
+				 (cpp_token *) expanded_beg
+				 : (cpp_token **) expanded_end -
+				 (cpp_token **) expanded_beg,
+				 t->direct_p);
+	    }

+	  if (pfile->cb.macro_end_arg)
+	    pfile->cb.macro_end_arg (pfile, false);
 	  if (macro->paramc > 0)
 	    replace_args (pfile, node, macro, (macro_arg *) buff->base);
 	  _cpp_release_buff (pfile, buff);
@@ -1253,6 +1272,8 @@ cpp_get_token (cpp_reader *pfile)
 	}
       else
 	{
+	  if (pfile->cb.macro_end_expand)
+	    pfile->cb.macro_end_expand (pfile, pfile.context.macro);
 	  _cpp_pop_context (pfile);
 	  if (pfile->state.in_directive)
 	    continue;
@@ -1310,7 +1331,14 @@ cpp_get_token (cpp_reader *pfile)
 		}
 	    }
 	  else
-	    ret = enter_macro_context (pfile, node, result);
+ 	    {
+	      if (pfile->cb.macro_start_expand)
+	        pfile->cb.macro_start_expand (pfile, result, node);
+	      ret = enter_macro_context (pfile, node, result);
+	      if (ret == 0 && pfile->cb.macro_end_expand)
+	        /* macro expansion is canceled. */
+	        pfile->cb.macro_end_expand (pfile, node);
+ 	    }
 	  if (ret)
  	    {
 	      if (pfile->state.in_directive || ret == 2)
Index: gcc/libcpp/lex.c
===================================================================
--- gcc.orig/libcpp/lex.c
+++ gcc/libcpp/lex.c
@@ -1729,6 +1729,8 @@ _cpp_lex_direct (cpp_reader *pfile)
       break;
     }

+  if (pfile->cb.directive_token)
+    pfile->cb.directive_token (pfile, result);
   return result;
 }


Patch 2 of 2 on gcc/
-------------------------
Index: gcc/gcc/c-lex.c
===================================================================
--- gcc.orig/gcc/c-lex.c
+++ gcc/gcc/c-lex.c
@@ -41,6 +41,7 @@ along with GCC; see the file COPYING3.
 #include "splay-tree.h"
 #include "debug.h"
 #include "target.h"
+#include "plugin.h"

 /* We may keep statistics about how long which files took to compile.  */
 static int header_time, body_time;
@@ -300,10 +301,13 @@ c_lex_with_flags (tree *value, location_
   const cpp_token *tok;
   enum cpp_ttype type;
   unsigned char add_flags = 0;
+  void *plugin_tok;

   timevar_push (TV_CPP);
  retry:
   tok = cpp_get_token_with_location (parse_in, loc);
+  plugin_tok = &tok;
+  invoke_plugin_callbacks (PLUGIN_CPP_TOKEN, *(cpp_token **) plugin_tok);
   type = tok->type;

  retry_after_at:
Index: gcc/gcc/doc/plugins.texi
===================================================================
--- gcc.orig/gcc/doc/plugins.texi
+++ gcc/gcc/doc/plugins.texi
@@ -176,6 +176,7 @@ enum plugin_event
   PLUGIN_EARLY_GIMPLE_PASSES_END,
   /* Called when a pass is first instantiated.  */
   PLUGIN_NEW_PASS,
+  PLUGIN_CPP_TOKEN,                /* Called when GCC gets a cpp token. */

   PLUGIN_EVENT_FIRST_DYNAMIC    /* Dummy event used for indexing callback
                                    array.  */
Index: gcc/gcc/plugin.c
===================================================================
--- gcc.orig/gcc/plugin.c
+++ gcc/gcc/plugin.c
@@ -418,6 +418,7 @@ register_callback (const char *plugin_na
       case PLUGIN_EARLY_GIMPLE_PASSES_START:
       case PLUGIN_EARLY_GIMPLE_PASSES_END:
       case PLUGIN_NEW_PASS:
+      case PLUGIN_CPP_TOKEN:
         {
           struct callback_info *new_callback;
           if (!callback)
@@ -499,6 +500,7 @@ invoke_plugin_callbacks (int event, void
       case PLUGIN_EARLY_GIMPLE_PASSES_START:
       case PLUGIN_EARLY_GIMPLE_PASSES_END:
       case PLUGIN_NEW_PASS:
+      case PLUGIN_CPP_TOKEN:
         {
           /* Iterate over every callback registered with this event and
              call it.  */
Index: gcc/gcc/plugin.def
===================================================================
--- gcc.orig/gcc/plugin.def
+++ gcc/gcc/plugin.def
@@ -89,6 +89,9 @@ DEFEVENT (PLUGIN_EARLY_GIMPLE_PASSES_END
 /* Called when a pass is first instantiated.  */
 DEFEVENT (PLUGIN_NEW_PASS)

+/* Called when a cpp token is extracted.  */
+DEFEVENT (PLUGIN_CPP_TOKEN)
+
 /* After the hard-coded events above, plugins can dynamically allocate events
    at run time.
    PLUGIN_EVENT_FIRST_DYNAMIC only appears as last enum element.  */

Yunfeng ZHANG