Fix WHOPR file name reuse

[Jan Hubicka <hubicka at ucw dot cz>]

Hi,
with this patch I can bootstrap with -fwhopr=n.  Main limitation is that I can not
use -j n for Makefile since file names used for ltrans partitions can get reused.

The problem solved here is that get_filename_for_set can return same file name
for multiple units that leads to compilation failure.

The filenames based on concatenation of units that are parts of partition seems
ugly and long.   For normal whopr I guess we can go for ordinary temp
filenames, for debugging and incremental whopr I need however stable ltrans
names.

In near future I expect we will have several partitining algorithm - the
current stupid 1_to_1 for testsuite, smarter source file based for incremental
compilation (that one could split overly large units, merge small units and do
some shuffling of inlined functions and datastructures to reduce boundries) and
graph partitioning based minimizing ltrans boundaries and producing ltrans
units of similar sizes for final build.

This patch does not implement that but gets in this direction by adding
a structure describing partition.  I plan to add size info there so we can
sort partitions by size etc.  Partitioner now decides what name to use
and for 1-1 it simply goes for input file name.

One problem is that for libraries we end up with many input files called
"libbackend", so I added code to lto_wpa_write_files to add nunmbered
suffixes.  It would be nice to get back to original .o file or perhaps
just pick up source file name from debug info for better stability,
but this can be handled incrementally.

We will also need to concatenate partition with output file name or something
to make it safe wrt parallel build of multiple binaries (as it happens during
GCC bootstrap).

With this whopr bootstrap gets to stage 3, but the cc1 binary is miscompiled.
Looking at spec2k6 results, there are still some miscompilation after fixing
the ipacp problem, I will analyze these now.

Bootstrapped/regtested x86_64-linux, OK?

Honza

	* lto/lto.c: Do not define bitmap vector.
	(ltrans_partition_def): New structure.
	(ltrans_partition): New type and vector.
	(new_partition): New function
	(lto_1_to_1_map): Reorganize for partition structure.
	(lto_promote_cross_file_statics): Use partition vector.
	(prefix_name_with_star): Remove.
	(strip_extension): Be ready for .a extensions.
	(get_filename_for_partition): New function.
	(get_filename_for_set): Remove.
	(htab_str_eq): New function.
	(lto_wpa_write_files): Resolve file name conflicts; use partition vector.
Index: lto/lto.c
===================================================================
--- lto/lto.c	(revision 159936)
+++ lto/lto.c	(working copy)
@@ -58,10 +58,6 @@
 # define O_BINARY 0
 #endif
 
-
-DEF_VEC_P(bitmap);
-DEF_VEC_ALLOC_P(bitmap,heap);
-
 static GTY(()) tree first_personality_decl;
 
 
@@ -513,11 +509,32 @@
 #endif
 }
 
-/* Vector of all cgraph node sets. */
-static GTY (()) VEC(cgraph_node_set, gc) *lto_cgraph_node_sets;
-static GTY (()) VEC(varpool_node_set, gc) *lto_varpool_node_sets;
+/* Structure describing ltrans partitions.  */
 
+struct GTY (()) ltrans_partition_def
+{
+  cgraph_node_set cgraph_set;
+  varpool_node_set varpool_set;
+  const char * GTY ((skip)) name;
+};
 
+typedef struct ltrans_partition_def *ltrans_partition;
+DEF_VEC_P(ltrans_partition);
+DEF_VEC_ALLOC_P(ltrans_partition,gc);
+
+static GTY (()) VEC(ltrans_partition, gc) *ltrans_partitions;
+
+static ltrans_partition
+new_partition (const char *filename)
+{
+  ltrans_partition part = GGC_NEW (struct ltrans_partition_def);
+  part->cgraph_set = cgraph_node_set_new ();
+  part->varpool_set = varpool_node_set_new ();
+  part->name = filename;
+  VEC_safe_push (ltrans_partition, gc, ltrans_partitions, part);
+  return part;
+}
+
 /* Group cgrah nodes by input files.  This is used mainly for testing
    right now.  */
 
@@ -528,18 +545,13 @@
   struct varpool_node *vnode;
   struct lto_file_decl_data *file_data;
   struct pointer_map_t *pmap;
-  struct pointer_map_t *vpmap;
-  cgraph_node_set set;
-  varpool_node_set vset;
+  ltrans_partition partition;
   void **slot;
+  int npartitions = 0;
 
   timevar_push (TV_WHOPR_WPA);
 
-  lto_cgraph_node_sets = VEC_alloc (cgraph_node_set, gc, 1);
-  lto_varpool_node_sets = VEC_alloc (varpool_node_set, gc, 1);
-
   pmap = pointer_map_create ();
-  vpmap = pointer_map_create ();
 
   for (node = cgraph_nodes; node; node = node->next)
     {
@@ -555,61 +567,47 @@
 
       slot = pointer_map_contains (pmap, file_data);
       if (slot)
-	set = (cgraph_node_set) *slot;
+	partition = (ltrans_partition) *slot;
       else
 	{
-	  set = cgraph_node_set_new ();
+	  partition = new_partition (file_data->file_name);
 	  slot = pointer_map_insert (pmap, file_data);
-	  *slot = set;
-	  VEC_safe_push (cgraph_node_set, gc, lto_cgraph_node_sets, set);
-	  vset = varpool_node_set_new ();
-	  slot = pointer_map_insert (vpmap, file_data);
-	  *slot = vset;
-	  VEC_safe_push (varpool_node_set, gc, lto_varpool_node_sets, vset);
+	  *slot = partition;
+	  npartitions++;
 	}
 
-      cgraph_node_set_add (set, node);
+      cgraph_node_set_add (partition->cgraph_set, node);
     }
 
   for (vnode = varpool_nodes; vnode; vnode = vnode->next)
     {
       if (vnode->alias || !vnode->needed)
 	continue;
-      slot = pointer_map_contains (vpmap, file_data);
+      slot = pointer_map_contains (pmap, file_data);
       if (slot)
-	vset = (varpool_node_set) *slot;
+	partition = (ltrans_partition) *slot;
       else
 	{
-	  set = cgraph_node_set_new ();
+	  partition = new_partition (file_data->file_name);
 	  slot = pointer_map_insert (pmap, file_data);
-	  *slot = set;
-	  VEC_safe_push (cgraph_node_set, gc, lto_cgraph_node_sets, set);
-	  vset = varpool_node_set_new ();
-	  slot = pointer_map_insert (vpmap, file_data);
-	  *slot = vset;
-	  VEC_safe_push (varpool_node_set, gc, lto_varpool_node_sets, vset);
+	  *slot = partition;
+	  npartitions++;
 	}
 
-      varpool_node_set_add (vset, vnode);
+      varpool_node_set_add (partition->varpool_set, vnode);
     }
 
   /* If the cgraph is empty, create one cgraph node set so that there is still
      an output file for any variables that need to be exported in a DSO.  */
-  if (!lto_cgraph_node_sets)
-    {
-      set = cgraph_node_set_new ();
-      VEC_safe_push (cgraph_node_set, gc, lto_cgraph_node_sets, set);
-      vset = varpool_node_set_new ();
-      VEC_safe_push (varpool_node_set, gc, lto_varpool_node_sets, vset);
-    }
+  if (!npartitions)
+    new_partition ("empty");
 
   pointer_map_destroy (pmap);
-  pointer_map_destroy (vpmap);
 
   timevar_pop (TV_WHOPR_WPA);
 
-  lto_stats.num_cgraph_partitions += VEC_length (cgraph_node_set, 
-						 lto_cgraph_node_sets);
+  lto_stats.num_cgraph_partitions += VEC_length (ltrans_partition, 
+						 ltrans_partitions);
 }
 
 
@@ -751,11 +749,12 @@
 
   gcc_assert (flag_wpa);
 
-  n_sets = VEC_length (cgraph_node_set, lto_cgraph_node_sets);
+  n_sets = VEC_length (ltrans_partition, ltrans_partitions);
   for (i = 0; i < n_sets; i++)
     {
-      set = VEC_index (cgraph_node_set, lto_cgraph_node_sets, i);
-      vset = VEC_index (varpool_node_set, lto_varpool_node_sets, i);
+      ltrans_partition part = VEC_index (ltrans_partition, ltrans_partitions, i);
+      set = part->cgraph_set;
+      vset = part->varpool_set;
 
       /* If node has either address taken (and we have no clue from where)
 	 or it is called from other partition, it needs to be globalized.  */
@@ -848,32 +847,27 @@
 }
 
 
-/* Given a file name FNAME, return a string with FNAME prefixed with '*'.  */
+/* Return a copy of FNAME without the .o or .a extension (if present).  */
 
 static char *
-prefix_name_with_star (const char *fname)
-{
-  char *star_fname;
-  size_t len;
-  
-  len = strlen (fname) + 1 + 1;
-  star_fname = XNEWVEC (char, len);
-  snprintf (star_fname, len, "*%s", fname);
-
-  return star_fname;
-}
-
-
-/* return a copy of FNAME without the .o extension.  */
-
-static char *
 strip_extension (const char *fname)
 {
-  char *s = XNEWVEC (char, strlen (fname) - 2 + 1);
-  gcc_assert (strstr (fname, ".o"));
-  snprintf (s, strlen (fname) - 2 + 1, "%s", fname);
+  int len = strlen (fname);
+  if (len < 2
+      || (fname[len - 1] != 'o' && fname[len - 1]  != 'a')
+      || fname[len - 2] != '.')
+    {
+      char *s = XNEWVEC (char, len + 1);
+      strcpy (s, fname);
+      return s;
+    }
+  else
+    {
+      char *s = XNEWVEC (char, len - 2 + 1);
+      snprintf (s, len - 2 + 1, "%s", fname);
+      return s;
+    }
 
-  return s;
 }
 
 
@@ -883,77 +877,35 @@
    the same input file.  */
 
 static char *
-get_filename_for_set (cgraph_node_set set)
+get_filename_for_partition (ltrans_partition part, int index)
 {
   char *fname = NULL;
-  static const size_t max_fname_len = 100;
+  char suffix[16];
 
-  /* Create a new temporary file to store SET.  To facilitate
-     debugging, use file names from SET as part of the new
-     temporary file name.  */
-  cgraph_node_set_iterator si;
-  struct pointer_set_t *pset = pointer_set_create ();
-  for (si = csi_start (set); !csi_end_p (si); csi_next (&si))
-    {
-      struct cgraph_node *n = csi_node (si);
-      const char *node_fname;
-      char *f;
+  fname = strip_extension (part->name);
 
-      /* Don't use the same file name more than once.  */
-      if (pointer_set_insert (pset, n->local.lto_file_data))
-	continue;
-
-      /* The first file name found in SET determines the output
-	 directory.  For the remaining files, we use their
-	 base names.  */
-      node_fname = n->local.lto_file_data->file_name;
-      if (fname == NULL)
-	{
-	  fname = strip_extension (node_fname);
-	  continue;
-	}
-
-      f = strip_extension (lbasename (node_fname));
-
-      /* If the new name causes an excessively long file name,
-	 make the last component "___" to indicate overflow.  */
-      if (strlen (fname) + strlen (f) > max_fname_len - 3)
-	{
-	  fname = reconcat (fname, fname, "___", NULL);
-	  break;
-	}
-      else
-	{
-	  fname = reconcat (fname, fname, "_", f, NULL);
-	  free (f);
-	}
-    }
-
-  pointer_set_destroy (pset);
-
-  if (!fname)
-    {
-      /* Since SET does not need to be processed by LTRANS, use
-	 the original file name and mark it with a '*' prefix so that
-	 lto_execute_ltrans knows not to process it.  */
-      cgraph_node_set_iterator si = csi_start (set);
-      struct cgraph_node *first = csi_node (si);
-      fname = prefix_name_with_star (first->local.lto_file_data->file_name);
-    }
+  /* Add the extension .wpa.o to indicate that this file has been
+     produced by WPA.  */
+  if (index)
+    sprintf (suffix, ".%i.wpa.o", index);
   else
-    {
-      /* Add the extension .wpa.o to indicate that this file has been
-	 produced by WPA.  */
-      fname = reconcat (fname, fname, ".wpa.o", NULL);
-      gcc_assert (fname);
-    }
+    sprintf (suffix, ".wpa.o");
+  fname = reconcat (fname, fname, suffix, NULL);
+  gcc_assert (fname);
 
   return fname;
 }
 
 static lto_file *current_lto_file;
 
+/* Helper function for the hash table that compares file names.  */
 
+static int
+htab_str_eq (const void *s1, const void *s2)
+{
+  return !strcmp ((const char *)s1, (const char *) s2);
+}
+
 /* Write all output files in WPA mode.  Returns a NULL-terminated array of
    output file names.  */
 
@@ -965,6 +917,11 @@
   lto_file *file;
   cgraph_node_set set;
   varpool_node_set vset;
+  ltrans_partition part;
+  htab_t name_hash = htab_create (VEC_length (ltrans_partition,
+					      ltrans_partitions),
+				  htab_hash_string, htab_str_eq,
+				  NULL);
 
   timevar_push (TV_WHOPR_WPA);
 
@@ -972,11 +929,11 @@
      compiled by LTRANS.  After this loop, only those sets that
      contain callgraph nodes from more than one file will need to be
      compiled by LTRANS.  */
-  for (i = 0; VEC_iterate (cgraph_node_set, lto_cgraph_node_sets, i, set); i++)
+  for (i = 0; VEC_iterate (ltrans_partition, ltrans_partitions, i, part); i++)
     {
-      lto_add_all_inlinees (set);
+      lto_add_all_inlinees (part->cgraph_set);
       lto_stats.num_output_cgraph_nodes += VEC_length (cgraph_node_ptr,
-						       set->nodes);
+						       part->cgraph_set->nodes);
     }
 
   /* After adding all inlinees, find out statics that need to be promoted
@@ -990,39 +947,57 @@
   /* The number of output files depends on the number of input files
      and how many callgraph node sets we create.  Reserve enough space
      for the maximum of these two.  */
-  num_out_files = MAX (VEC_length (cgraph_node_set, lto_cgraph_node_sets),
+  num_out_files = MAX (VEC_length (ltrans_partition, ltrans_partitions),
                        num_in_fnames);
   output_files = XNEWVEC (char *, num_out_files + 1);
 
-  n_sets = VEC_length (cgraph_node_set, lto_cgraph_node_sets);
+  n_sets = VEC_length (ltrans_partition, ltrans_partitions);
   for (i = 0; i < n_sets; i++)
     {
       char *temp_filename;
+      ltrans_partition part = VEC_index (ltrans_partition, ltrans_partitions, i);
+      unsigned int index = 0;
+      bool found_match;
 
-      set = VEC_index (cgraph_node_set, lto_cgraph_node_sets, i);
-      vset = VEC_index (varpool_node_set, lto_varpool_node_sets, i);
-      temp_filename = get_filename_for_set (set);
+      set = part->cgraph_set;
+      vset = part->varpool_set;
+      do
+	{
+	  void **slot;
+	  temp_filename = get_filename_for_partition (part, index);
+	  slot = htab_find_slot (name_hash, temp_filename, INSERT);
+	  if (*slot)
+	    {
+	      found_match = true;
+	      index++;
+	    }
+	  else
+	    {
+	      found_match = false;
+	      *slot = temp_filename;
+	    }
+	}
+      while (found_match);
       output_files[i] = temp_filename;
 
-      if (cgraph_node_set_nonempty_p (set) || varpool_node_set_nonempty_p (vset))
-	{
-	  /* Write all the nodes in SET to TEMP_FILENAME.  */
-	  file = lto_obj_file_open (temp_filename, true);
-	  if (!file)
-	    fatal_error ("lto_obj_file_open() failed");
+      gcc_assert (cgraph_node_set_nonempty_p (set) || varpool_node_set_nonempty_p (vset));
+      /* Write all the nodes in SET to TEMP_FILENAME.  */
+      file = lto_obj_file_open (temp_filename, true);
+      if (!file)
+	fatal_error ("lto_obj_file_open() failed");
 
-	  if (!quiet_flag)
-	    fprintf (stderr, " %s", temp_filename);
+      if (!quiet_flag)
+	fprintf (stderr, " %s", temp_filename);
 
-	  lto_set_current_out_file (file);
+      lto_set_current_out_file (file);
 
-	  ipa_write_optimization_summaries (set, vset);
+      ipa_write_optimization_summaries (set, vset);
 
-	  lto_set_current_out_file (NULL);
-	  lto_obj_file_close (file);
-	}
+      lto_set_current_out_file (NULL);
+      lto_obj_file_close (file);
     }
 
+  htab_delete (name_hash);
   last_out_file_ix = n_sets;
 
   lto_stats.num_output_files += n_sets;