[incremental] Patch: FYI: compile server -vs- mapped locations

[Tom Tromey <tromey at redhat dot com>]

I'm checking this in on the incremental-compiler branch.

The move to mapped locations created a problem for the compile server:
it is possible to run out of location numbers in a long-lived compile.

This patch is the first draft of a fix for this problem.  After
compilation, we turn a decl's location into an offset into the hunk
from which the decl came.  Then, when mapping a hunk into the current
compilation, we use the location of the corresponding token in the new
token stream.  This works because a hunk's size (in tokens) is
constant.  Then, it is a simple matter to reset the line table before
each compilation.

More work remains to be done here; this patch only resets the
locations of decls created in a hunk.  However, ideally it should also
handle function bodies and variable initializer expressions.

In order to use a binary search to find the token offset, I had to
ensure that the locations emitted by libcpp were monotonically
nondecreasing.  At first I assumed this was always true, but then I
discovered various cases where libcpp cheats on its location
computations.  So, this includes an ugly patch to libcpp to make
things better.

Also, this adds an iterator for hash tables.  On irc I was told about
the code in tree-flow.h -- after writing my implementation.  I prefer
mine, since it is simpler and it lets you remove an entry while
iterating (though the tree-flow stuff is nicer type-wise.  Perhaps
they should be merged; in any case this stuff belongs in hashtab.h,
not somewhere random like tree-flow.h).

I tested this by configuring and building gdb using the compile server
for all compilations.

Tom

include/ChangeLog:
2008-04-23  Tom Tromey  <tromey@redhat.com>

	* hashtab.h (htab_iterate): New function.

libcpp/ChangeLog:
2008-04-23  Tom Tromey  <tromey@redhat.com>

	* macro.c (builtin_macro): Set paste_p.
	(paste_all_tokens): Likewise.
	(funlike_invocation_p): Likewise.
	(next_context): Clear paste_p.
	(cpp_get_token): Use paste_p.
	(cpp_get_token_with_location): Likewise.
	* internal.h (struct cpp_context) <paste_p>: New field.

gcc/ChangeLog:
2008-04-23  Tom Tromey  <tromey@redhat.com>

	* toplev.c (create_line_map): New function.
	(general_init): Use it.
	(compile_file): Call server_steady_state hook.
	(server_callback): Call create_line_map.
	* c-tree.h (c_server_steady_state): Declare.
	* c-lang.c (LANG_HOOKS_SERVER_STEADY_STATE): New define.
	* langhooks-def.h (LANG_HOOKS_SERVER_STEADY_STATE): New define.
	(LANG_HOOKS_INITIALIZER): Update.
	* langhooks.h (struct lang_hooks) <server_steady_state>: New
	function.
	* c-parser.c (traverse_hunk_binding_entry): Reset decl locations.
	(can_reuse_hunk): Pass parser to traverse_hunk_binding_entry.
	Reset binding's location and token offset.
	(struct hunk_binding) <token_offset, token_count>: New fields.
	(start_new_parsed_hunk): Set token_offset field.
	(c_server_steady_state): New function.
	(universalize_decl_locations): Likewise.
	(c_parser_create_smash_map): Null 'the_parser'.
	(c_parse_file): Don't null 'the_parser'.
	(reset_tree_location): New function.
	(universalize_tree): Likewise.

Index: include/hashtab.h
===================================================================
--- include/hashtab.h	(revision 132956)
+++ include/hashtab.h	(working copy)
@@ -1,5 +1,5 @@
 /* An expandable hash tables datatype.  
-   Copyright (C) 1999, 2000, 2002, 2003, 2004 Free Software Foundation, Inc.
+   Copyright (C) 1999, 2000, 2002, 2003, 2004, 2008 Free Software Foundation, Inc.
    Contributed by Vladimir Makarov (vmakarov@cygnus.com).
 
 This program is free software; you can redistribute it and/or modify
@@ -199,6 +199,36 @@
 /* Shorthand for hashing something with an intrinsic size.  */
 #define iterative_hash_object(OB,INIT) iterative_hash (&OB, sizeof (OB), INIT)
 
+/* Iteration step for a hash table.  Updates '*INDEX' and modifies
+   '*SLOTP' to point to the next available slot.  Returns false when
+   finished; true otherwise.  Should be used like:
+   
+   size_t index;
+   void **slot;
+   for (index = 0; htab_iterate (table, &index, &slot); ++index)
+     content;
+*/
+static inline int
+htab_iterate (htab_t htab, size_t *index, void ***slotp)
+{
+  size_t i = *index;
+  PTR *slot = &htab->entries[i];
+  PTR *limit = htab->entries + htab->size;
+  while (slot < limit)
+    {
+      PTR x = *slot;
+      if (x != HTAB_EMPTY_ENTRY && x != HTAB_DELETED_ENTRY)
+	{
+	  *index = i;
+	  *slotp = slot;
+	  return 1;
+	}
+      ++i;
+      ++slot;
+    }
+  return 0;
+}
+
 #ifdef __cplusplus
 }
 #endif /* __cplusplus */
Index: libcpp/macro.c
===================================================================
--- libcpp/macro.c	(revision 132956)
+++ libcpp/macro.c	(working copy)
@@ -318,6 +318,7 @@
   /* Set pfile->cur_token as required by _cpp_lex_direct.  */
   pfile->cur_token = _cpp_temp_token (pfile);
   _cpp_push_token_context (pfile, NULL, _cpp_lex_direct (pfile), 1);
+  pfile->context->paste_p = 1;
   if (pfile->buffer->cur != pfile->buffer->rlimit)
     cpp_error (pfile, CPP_DL_ICE, "invalid built-in macro \"%s\"",
 	       NODE_NAME (node));
@@ -530,6 +531,7 @@
 
   /* Put the resulting token in its own context.  */
   _cpp_push_token_context (pfile, NULL, lhs, 1);
+  pfile->context->paste_p = true;
 }
 
 /* Returns TRUE if the number of arguments ARGC supplied in an
@@ -793,7 +795,11 @@
 	 too difficult.  We re-insert it in its own context.  */
       _cpp_backup_tokens (pfile, 1);
       if (padding)
-	_cpp_push_token_context (pfile, NULL, padding, 1);
+	{
+	  bool paste_p = pfile->context->macro || pfile->context->paste_p;
+	  _cpp_push_token_context (pfile, NULL, padding, 1);
+	  pfile->context->paste_p = paste_p;
+	}
     }
 
   return NULL;
@@ -1050,6 +1056,7 @@
       pfile->context->next = result;
     }
 
+  result->paste_p = 0;
   pfile->context = result;
   return result;
 }
@@ -1227,7 +1234,7 @@
 	  int ret;
 	  /* If not in a macro context, and we're going to start an
 	     expansion, record the location.  */
-	  if (can_set && !context->macro)
+	  if (can_set && !context->macro && !context->paste_p)
 	    pfile->invocation_location = result->src_loc;
 	  if (pfile->state.prevent_expansion)
 	    break;
@@ -1269,7 +1276,9 @@
 
   pfile->set_invocation_location = true;
   result = cpp_get_token (pfile);
-  if (pfile->context->macro)
+  if (pfile->context->macro
+      || pfile->context->paste_p
+      || pfile->invocation_location > result->src_loc)
     *loc = pfile->invocation_location;
   else
     *loc = result->src_loc;
Index: libcpp/internal.h
===================================================================
--- libcpp/internal.h	(revision 132956)
+++ libcpp/internal.h	(working copy)
@@ -171,6 +171,10 @@
 
   /* True if utoken element is token, else ptoken.  */
   bool direct_p;
+  /* True if the context is the result of token pasting, a builtin
+     macro, or other situation requiring special location
+     handling.  */
+  bool paste_p;
 };
 
 struct lexer_state
Index: gcc/toplev.c
===================================================================
--- gcc/toplev.c	(revision 134446)
+++ gcc/toplev.c	(working copy)
@@ -1131,7 +1131,10 @@
       ggc_collect ();
 
       if (server_start_back_end (&server_back_end_status))
-	return false;
+	{
+	  lang_hooks.server_steady_state ();
+	  return false;
+	}
 
       job->as_process = start_as ((char **) VEC_address (cchar_p,
 							 job->as_arguments));
@@ -1723,6 +1726,19 @@
   /* Do nothing, the GC will handle it.  */
 }
 
+static void
+create_line_map (void)
+{
+  location_t loc;
+  line_table = GGC_NEW (struct line_maps);
+  linemap_init (line_table);
+  line_table->reallocator = realloc_for_line_map;
+  line_table->freer = free_for_line_map;
+  linemap_add (line_table, LC_RENAME, 0, 0, _("<built-in>"), 0);
+  loc = linemap_line_start (line_table, 0, 1);
+  gcc_assert (loc == BUILTINS_LOCATION);
+}
+
 /* Initialization of the front end environment, before command line
    options are parsed.  Signal handlers, internationalization etc.
    ARGV0 is main's argv[0].  */
@@ -1730,7 +1746,6 @@
 general_init (const char *argv0)
 {
   const char *p;
-  location_t loc;
 
   p = argv0 + strlen (argv0);
   while (p != argv0 && !IS_DIR_SEPARATOR (p[-1]))
@@ -1781,13 +1796,7 @@
   init_ggc ();
   ggc_thread_init ();
   init_stringpool ();
-  line_table = GGC_NEW (struct line_maps);
-  linemap_init (line_table);
-  line_table->reallocator = realloc_for_line_map;
-  line_table->freer = free_for_line_map;
-  linemap_add (line_table, LC_RENAME, 0, 0, _("<built-in>"), 0);
-  loc = linemap_line_start (line_table, 0, 1);
-  gcc_assert (loc == BUILTINS_LOCATION);
+  create_line_map ();
 
   init_ttree ();
 
@@ -2571,6 +2580,11 @@
   copy_to_vec (&job->as_arguments, &job->strings, as_argv, NULL);
   copy_to_vec (&job->cc1_arguments, &job->strings, cc1_argv, &n);
 
+  /* Re-initialize the line table.  Note that this must be done before
+     decoding the options, so that lang hooks run during option
+     decoding can use the line table.  */
+  create_line_map ();
+
   decode_options (n, VEC_address (cchar_p, job->cc1_arguments));
   flag_unit_at_a_time = 1;
 
Index: gcc/c-tree.h
===================================================================
--- gcc/c-tree.h	(revision 133760)
+++ gcc/c-tree.h	(working copy)
@@ -459,6 +459,7 @@
 extern tree c_parser_find_binding (tree);
 extern void c_parser_lookup_callback (tree, tree, bool);
 extern void c_parse_file_wrapper (int);
+extern void c_server_steady_state (void);
 
 extern void c_parser_print_job_statistics (void);
 
Index: gcc/c-decl.c
===================================================================
--- gcc/c-decl.c	(revision 133841)
+++ gcc/c-decl.c	(working copy)
@@ -2129,12 +2129,14 @@
 	{
 	  /* Modify a copy of OLDDECL and install that in the
 	     bindings.  */
+	  location_t saved_loc = DECL_SOURCE_LOCATION (newdecl);
 	  tree copy = copy_node (olddecl);
 	  merge_decls (newdecl, copy, newtype,
 		       C_SMASHED_TYPE_VARIANT (oldtype));
 	  /* FIXME: this triggers building libgcc.  */
 	  /* 	  gcc_assert (binding->decl == olddecl); */
 	  C_FIRST_DEFINITION_P (copy) = C_FIRST_DEFINITION_P (newdecl);
+	  DECL_SOURCE_LOCATION (copy) = saved_loc;
 	  binding->decl = copy;
 
 	  /* Need to call decl_attributes on the new decl, to ensure
Index: gcc/langhooks.h
===================================================================
--- gcc/langhooks.h	(revision 132956)
+++ gcc/langhooks.h	(working copy)
@@ -1,5 +1,5 @@
 /* The lang_hooks data structure.
-   Copyright 2001, 2002, 2003, 2004, 2005, 2006, 2007
+   Copyright 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008
    Free Software Foundation, Inc.
 
 This file is part of GCC.
@@ -300,6 +300,12 @@
   /* Called immediately after parsing to clear the binding stack.  */
   void (*clear_binding_stack) (void);
 
+  /* Called only in server mode, after the server has forked for code
+     generation and before it returns to listening for client
+     connections.  This is intended to let the front end modify its
+     data structures to return to a "neutral" state, if needed.  */
+  void (*server_steady_state) (void);
+
   /* Called to obtain the alias set to be used for an expression or type.
      Returns -1 if the language does nothing special for it.  */
   alias_set_type (*get_alias_set) (tree);
Index: gcc/c-parser.c
===================================================================
--- gcc/c-parser.c	(revision 133760)
+++ gcc/c-parser.c	(working copy)
@@ -736,17 +736,37 @@
   return hbe_a->name == hbe_b->name;
 }
 
+/* Reset the location of a single tree.  */
+static void
+reset_tree_location (tree decl, c_parser *parser)
+{
+  /* FIXME: eventually operate on expressions and more.  */
+  if (TREE_CODE_CLASS (TREE_CODE (decl)) != tcc_declaration)
+    return;
+  DECL_SOURCE_LOCATION (decl) = parser->buffer[DECL_SOURCE_LOCATION (decl)
+					       + parser->next_token].location;
+}
+
 /* Called to map the contents of a struct hunk_binding_entry into the
-   current symbol table.  */
+   current symbol table.  This also resets the location of decls
+   declared in this hunk so that they reflect the current compilation
+   unit.  */
 static int
-traverse_hunk_binding_entry (void **slot, void *ignore ATTRIBUTE_UNUSED)
+traverse_hunk_binding_entry (void **slot, void *pp)
 {
   struct hunk_binding_entry *hb = (struct hunk_binding_entry *) *slot;
+  c_parser *parser = (c_parser *) pp;
 
   if (hb->tag_binding != NULL_TREE)
-    c_decl_re_bind (hb->name, hb->tag_binding);
+    {
+      reset_tree_location (hb->tag_binding, parser);
+      c_decl_re_bind (hb->name, hb->tag_binding);
+    }
   if (hb->symbol_binding != NULL_TREE)
-    c_decl_re_bind (hb->name, hb->symbol_binding);
+    {
+      reset_tree_location (hb->symbol_binding, parser);
+      c_decl_re_bind (hb->name, hb->symbol_binding);
+    }
 
   return 1;
 }
@@ -764,6 +784,12 @@
   struct parsed_hunk *multi_list;
   /* Location of the first token in this hunk.  */
   location_t location;
+  /* The offset of the first token in this binding.  This is only
+     valid during and just after a given compilation, and is used to
+     "universalize" decl locations.  */
+  unsigned int token_offset;
+  /* The total number of tokens in this hunk.  */
+  unsigned int token_count;
   /* Map names to bindings.  */
   htab_t GTY ((param_is (struct hunk_binding_entry))) binding_map;
   /* Set of all prerequisite bindings.  */
@@ -1015,6 +1041,9 @@
 		       eq_hunk_binding_entry,
 		       NULL);
   parser->current_hunk_binding->location = location;
+  parser->current_hunk_binding->token_offset = parser->next_token;
+  /* We set this when closing the hunk.  */
+  parser->current_hunk_binding->token_count = 0;
 }
 
 /* Called when finished parsing a hunk.  Registers the parsed hunk
@@ -1043,6 +1072,9 @@
 						  parser->current_hunk_binding,
 						  INSERT);
   gcc_assert (!*slot);
+  /* Compute the number of tokens in this binding.  */
+  parser->current_hunk_binding->token_count
+    = parser->next_token - parser->current_hunk_binding->token_offset;
   *slot = parser->current_hunk_binding;
 
   /* Also note this hunk in used_hunks, for long-term storage.  */
@@ -1188,6 +1220,10 @@
   if (! global_smash_map)
     return NULL;
 
+  /* It is a bit lame to do this here, but we don't need the parser
+     any more, and there's not a better place to get rid of it.  */
+  the_parser = NULL;
+
   /* We could do this without allocating by rewriting the map in
      place.  */
   result = htab_create_ggc (htab_size (global_smash_map),
@@ -1395,6 +1431,10 @@
       c_lex_one_token (&buffer[pos]);
       buffer[pos].user_owned = lstate->user_owned;
 
+      gcc_assert (pos == 0
+		  || buffer[pos].type == CPP_EOF
+		  || buffer[pos].location >= buffer[pos - 1].location);
+
       /* If there was a file change event, it happened before this
 	 token.  */
       if (lstate->file_change && hunk_start != (size_t) -1)
@@ -2259,10 +2299,18 @@
 						  info.binding, INSERT);
   *slot = info.binding;
 
+  /* Reset the binding's token offset.  This is used when
+     "universalizing" decl locations after the compilation unit is
+     finished.  */
+  info.binding->token_offset = parser->next_token;
+  /* Reset the binding's location.  Note that this is a debugging
+     convenience, this location is not used for anything.  */
+  info.binding->location = parser->buffer[parser->next_token].location;
+
   /* Map in the bindings.  */
   htab_traverse_noresize (info.binding->binding_map,
 			  traverse_hunk_binding_entry,
-			  NULL);
+			  parser);
   if (update_parser)
     {
       parser->first_hunk = info.self_iter;
@@ -2281,6 +2329,77 @@
   return true;
 }
 
+/* Universalize a single tree's location.  */
+static void
+universalize_tree (tree decl, c_parser *parser,
+		   unsigned int token_offset, unsigned int token_count)
+{
+  unsigned int offset = 0, min, max;
+  location_t orig_loc;
+
+  /* FIXME: eventually operate on expressions and more.  */
+  if (TREE_CODE_CLASS (TREE_CODE (decl)) != tcc_declaration)
+    return;
+
+  /* Binary search for the location in the token buffer.  This ought
+     to be efficient enough, as most declarations contain few tokens.  */
+  min = token_offset;
+  max = token_offset + token_count;
+  orig_loc = DECL_SOURCE_LOCATION (decl);
+  while (min <= max)
+    {
+      unsigned int mid = (min + max) / 2;
+      location_t match = parser->buffer[mid].location;
+      if (orig_loc == match)
+	{
+	  offset = mid;
+	  break;
+	}
+      else if (orig_loc < match)
+	max = mid - 1;
+      else
+	min = mid + 1;
+    }
+  /* We must always find an answer, because the location of a decl
+     must come from a particular token in the current hunk.  */
+  gcc_assert (parser->buffer[offset].location == orig_loc);
+  DECL_SOURCE_LOCATION (decl) = (location_t) (offset - token_offset);
+}
+
+/* Universalize the locations of all declarations used in this
+   compilation unit.  */
+static void
+universalize_decl_locations (c_parser *parser)
+{
+  size_t index;
+  void **slot;
+
+  /* Loop over all the hunks used in this compilation.  */
+  for (index = 0; htab_iterate (parser->used_hunks, &index, &slot); ++index)
+    {
+      size_t bindex;
+      void **bslot;
+      struct hunk_binding *binding = *(struct hunk_binding **) slot;
+
+      /* Loop over all the bindings from this hunk.  */
+      for (bindex = 0;
+	   htab_iterate (binding->binding_map, &bindex, &bslot);
+	   ++bindex)
+	{
+	  struct hunk_binding_entry *entry
+	    = * (struct hunk_binding_entry **) bslot;
+
+	  if (entry->tag_binding != NULL_TREE)
+	    universalize_tree (entry->tag_binding, parser,
+			       binding->token_offset, binding->token_count);
+
+	  if (entry->symbol_binding != NULL_TREE)
+	    universalize_tree (entry->symbol_binding, parser,
+			       binding->token_offset, binding->token_count);
+	}
+    }
+}
+
 /* Parse a translation unit (C90 6.7, C99 6.9).
 
    translation-unit:
@@ -9566,7 +9685,12 @@
 
   /* Preserve the hunks created or used during this compilation.  */
   copy_used_hunks (the_parser->used_hunks);
+}
 
+void
+c_server_steady_state (void)
+{
+  universalize_decl_locations (the_parser);
   the_parser = NULL;
 }
 
Index: gcc/c-lang.c
===================================================================
--- gcc/c-lang.c	(revision 132956)
+++ gcc/c-lang.c	(working copy)
@@ -47,6 +47,8 @@
 #define LANG_HOOKS_CLEAR_BINDING_STACK c_clear_binding_stack
 #undef LANG_HOOKS_PARSE_FILE
 #define LANG_HOOKS_PARSE_FILE c_parse_file_wrapper
+#undef LANG_HOOKS_SERVER_STEADY_STATE
+#define LANG_HOOKS_SERVER_STEADY_STATE c_server_steady_state
 
 /* Each front end provides its own lang hook initializer.  */
 const struct lang_hooks lang_hooks = LANG_HOOKS_INITIALIZER;
Index: gcc/langhooks-def.h
===================================================================
--- gcc/langhooks-def.h	(revision 132956)
+++ gcc/langhooks-def.h	(working copy)
@@ -1,5 +1,5 @@
 /* Default macros to initialize the lang_hooks data structure.
-   Copyright 2001, 2002, 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc.
+   Copyright 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
    Contributed by Alexandre Oliva  <aoliva@redhat.com>
 
 This file is part of GCC.
@@ -87,6 +87,7 @@
 #define LANG_HOOKS_FINISH		lhd_do_nothing
 #define LANG_HOOKS_PARSE_FILE		lhd_do_nothing_i
 #define LANG_HOOKS_CLEAR_BINDING_STACK	lhd_do_nothing
+#define LANG_HOOKS_SERVER_STEADY_STATE lhd_do_nothing
 #define LANG_HOOKS_INIT_OPTIONS		hook_uint_uint_constcharptrptr_0
 #define LANG_HOOKS_INITIALIZE_DIAGNOSTICS lhd_initialize_diagnostics
 #define LANG_HOOKS_HANDLE_OPTION	hook_int_size_t_constcharptr_int_0
@@ -254,6 +255,7 @@
   LANG_HOOKS_FINISH, \
   LANG_HOOKS_PARSE_FILE, \
   LANG_HOOKS_CLEAR_BINDING_STACK, \
+  LANG_HOOKS_SERVER_STEADY_STATE, \
   LANG_HOOKS_GET_ALIAS_SET, \
   LANG_HOOKS_EXPAND_EXPR, \
   LANG_HOOKS_EXPAND_DECL, \