Re: Optimization summary streaming

[Richard Guenther <rguenther at suse dot de>]

On Tue, 20 Apr 2010, Jan Hubicka wrote:

> Hi,
> this patch adds hook to passmanager for optimization summary reading and writting.
> When doing compile->wpa->ltrans stages, we actually need to stream different
> info for IPA passes for compile->wpa (summaries of local properties) and
> wpa->ltrans (summaries of IPA propagation of the pass).
> Currently we sort of mix both together and have no way for IPA passes
> to pass anything from wpa->ltrans (just for WPA passes that are executed
> by hand separately).
> 
> The patch also adds some -Q messages into lto.c.  I find these useful when
> waiting for long compilations to converge to see what is going on and where
> time is going. (and it is similar to what we do with normal compilation
> anyway). In particular all the WPA time is no longer attributes to inliner ;)
> 
> bootstrapped/regtested x86_64-linux and tested with spec2000 gcc -whopr
> build, OK?

Ok.

Thanks,
Richard.

> Honza
> 
> 	* tree-pass.h (ipa_opt_pass_d): Rename function_read_summary;
> 	add write_optimization_summary, read_optimization_summary.
> 	(ipa_write_summaries_of_cgraph_node_set): Remove.
> 	(ipa_write_optimization_summaries): Declare.
> 	(ipa_read_optimization_summaries): Declare.
> 	* ipa-cp.c (pass_ipa_cp): Update.
> 	* ipa-reference.c (pass_ipa_reference): Update.
> 	* ipa-pure-const.c (pass_ipa_pure_const): Update.
> 	* lto-streamer-out.c (pass_ipa_lto_gimple, pass_ipa_lto_finish):
> 	Update.
> 	* ipa-inline.c (pass_ipa_inline): Update.
> 	* ipa.c (pass_ipa_whole_program): Update.
> 	* lto-wpa-fixup.c (pass_ipa_lto_wpa_fixup): Update.
> 	* passes.c (ipa_write_summaries_1): Do not test wpa.
> 	(ipa_write_optimization_summaries_1): New.
> 	(ipa_write_optimization_summaries): New.
> 	(ipa_read_summaries): Do not test ltrans.
> 	(ipa_read_optimization_summaries_1): New.
> 	(ipa_read_optimization_summaries): New.
> 
> 	* lto.c (lto_wpa_write_files): Update.
> 	(read_cgraph_and_symbols): Be more verbose.
> 	(materialize_cgraph): Likewise.
> 	(do_whole_program_analysis): Likewise.
> Index: tree-pass.h
> ===================================================================
> --- tree-pass.h	(revision 158563)
> +++ tree-pass.h	(working copy)
> @@ -182,11 +182,15 @@ struct ipa_opt_pass_d
>    /* This hook is used to serialize IPA summaries on disk.  */
>    void (*write_summary) (struct cgraph_node_set_def *);
>  
> -  /* For most ipa passes, the information can only be deserialized in
> -     one chunk.  However, function bodies are read function at a time
> -     as needed so both calls are necessary.  */
> +  /* This hook is used to deserialize IPA summaries from disk.  */
>    void (*read_summary) (void);
> -  void (*function_read_summary) (struct cgraph_node *);
> +
> +  /* This hook is used to serialize IPA optimization summaries on disk.  */
> +  void (*write_optimization_summary) (struct cgraph_node_set_def *);
> +
> +  /* This hook is used to deserialize IPA summaries from disk.  */
> +  void (*read_optimization_summary) (void);
> +
>    /* Hook to convert gimple stmt uids into true gimple statements.  The second
>       parameter is an array of statements indexed by their uid. */
>    void (*stmt_fixup) (struct cgraph_node *, gimple *);
> @@ -601,9 +605,9 @@ extern const char *get_current_pass_name
>  extern void print_current_pass (FILE *);
>  extern void debug_pass (void);
>  extern void ipa_write_summaries (void);
> -extern void ipa_write_summaries_of_cgraph_node_set (
> -	      struct cgraph_node_set_def *);
> +extern void ipa_write_optimization_summaries (struct cgraph_node_set_def *);
>  extern void ipa_read_summaries (void);
> +extern void ipa_read_optimization_summaries (void);
>  extern void register_one_dump_file (struct opt_pass *);
>  extern bool function_called_by_processed_nodes_p (void);
>  extern void register_pass (struct register_pass_info *);
> Index: ipa-cp.c
> ===================================================================
> --- ipa-cp.c	(revision 158563)
> +++ ipa-cp.c	(working copy)
> @@ -1344,7 +1344,8 @@ struct ipa_opt_pass_d pass_ipa_cp =
>   ipcp_generate_summary,			/* generate_summary */
>   ipcp_write_summary,			/* write_summary */
>   ipcp_read_summary,			/* read_summary */
> - NULL,					/* function_read_summary */
> + NULL,					/* write_optimization_summary */
> + NULL,					/* read_optimization_summary */
>   lto_ipa_fixup_call_notes, 		/* stmt_fixup */
>   0,					/* TODOs */
>   NULL,					/* function_transform */
> Index: ipa-reference.c
> ===================================================================
> --- ipa-reference.c	(revision 158563)
> +++ ipa-reference.c	(working copy)
> @@ -1514,7 +1514,8 @@ struct ipa_opt_pass_d pass_ipa_reference
>   generate_summary,		        /* generate_summary */
>   ipa_reference_write_summary,		/* write_summary */
>   ipa_reference_read_summary,		/* read_summary */
> - NULL,					/* function_read_summary */
> + NULL,					/* write_optimization_summary */
> + NULL,					/* read_optimization_summary */
>   NULL,					/* stmt_fixup */
>   0,					/* TODOs */
>   NULL,			                /* function_transform */
> Index: ipa-pure-const.c
> ===================================================================
> --- ipa-pure-const.c	(revision 158563)
> +++ ipa-pure-const.c	(working copy)
> @@ -1095,7 +1095,8 @@ struct ipa_opt_pass_d pass_ipa_pure_cons
>   generate_summary,		        /* generate_summary */
>   pure_const_write_summary,		/* write_summary */
>   pure_const_read_summary,		/* read_summary */
> - NULL,					/* function_read_summary */
> + NULL,					/* write_optimization_summary */
> + NULL,					/* read_optimization_summary */
>   NULL,					/* stmt_fixup */
>   0,					/* TODOs */
>   NULL,			                /* function_transform */
> Index: lto-streamer-out.c
> ===================================================================
> --- lto-streamer-out.c	(revision 158563)
> +++ lto-streamer-out.c	(working copy)
> @@ -2147,7 +2147,8 @@ struct ipa_opt_pass_d pass_ipa_lto_gimpl
>   NULL,		                        /* generate_summary */
>   lto_output,           			/* write_summary */
>   NULL,		         		/* read_summary */
> - NULL,					/* function_read_summary */
> + lto_output,           			/* write_optimization_summary */
> + NULL,					/* read_optimization_summary */
>   NULL,					/* stmt_fixup */
>   0,					/* TODOs */
>   NULL,			                /* function_transform */
> @@ -2565,7 +2566,8 @@ struct ipa_opt_pass_d pass_ipa_lto_finis
>   NULL,		                        /* generate_summary */
>   produce_asm_for_decls,			/* write_summary */
>   NULL,		         		/* read_summary */
> - NULL,					/* function_read_summary */
> + produce_asm_for_decls,			/* write_optimization_summary */
> + NULL,					/* read_optimization_summary */
>   NULL,					/* stmt_fixup */
>   0,					/* TODOs */
>   NULL,			                /* function_transform */
> Index: ipa-inline.c
> ===================================================================
> --- ipa-inline.c	(revision 158563)
> +++ ipa-inline.c	(working copy)
> @@ -2132,7 +2132,8 @@ struct ipa_opt_pass_d pass_ipa_inline =
>   inline_generate_summary,		/* generate_summary */
>   inline_write_summary,			/* write_summary */
>   inline_read_summary,			/* read_summary */
> - NULL,					/* function_read_summary */
> + NULL,					/* write_optimization_summary */
> + NULL,					/* read_optimization_summary */
>   lto_ipa_fixup_call_notes,		/* stmt_fixup */
>   0,					/* TODOs */
>   inline_transform,			/* function_transform */
> Index: ipa.c
> ===================================================================
> --- ipa.c	(revision 158563)
> +++ ipa.c	(working copy)
> @@ -271,6 +271,8 @@ cgraph_remove_unreachable_nodes (bool be
>  		      node->analyzed = false;
>  		      node->local.inlinable = false;
>  		    }
> +		  else
> +		    gcc_assert (!clone->in_other_partition);
>  		  cgraph_node_remove_callees (node);
>  		  if (node->prev_sibling_clone)
>  		    node->prev_sibling_clone->next_sibling_clone = node->next_sibling_clone;
> @@ -574,7 +576,8 @@ struct ipa_opt_pass_d pass_ipa_whole_pro
>   NULL,					/* generate_summary */
>   NULL,					/* write_summary */
>   NULL,					/* read_summary */
> - NULL,					/* function_read_summary */
> + NULL,					/* write_optimization_summary */
> + NULL,					/* read_optimization_summary */
>   NULL,					/* stmt_fixup */
>   0,					/* TODOs */
>   NULL,					/* function_transform */
> Index: lto-wpa-fixup.c
> ===================================================================
> --- lto-wpa-fixup.c	(revision 158563)
> +++ lto-wpa-fixup.c	(working copy)
> @@ -271,9 +271,10 @@ struct ipa_opt_pass_d pass_ipa_lto_wpa_f
>    TODO_dump_func                        /* todo_flags_finish */
>   },
>   NULL,		                        /* generate_summary */
> - lto_output_wpa_fixup,			/* write_summary */
> - lto_input_wpa_fixup,			/* read_summary */
> - NULL,					/* function_read_summary */
> + NULL,					/* write_summary */
> + NULL,					/* read_summary */
> + lto_output_wpa_fixup,			/* write_optimization_summary */
> + lto_input_wpa_fixup,			/* read_optimization_summary */
>   NULL,					/* stmt_fixup */
>   0,					/* TODOs */
>   NULL,			                /* function_transform */
> Index: lto/lto.c
> ===================================================================
> --- lto/lto.c	(revision 158563)
> +++ lto/lto.c	(working copy)
> @@ -1045,7 +1045,7 @@ lto_wpa_write_files (void)
>  
>  	  lto_set_current_out_file (file);
>  
> -	  ipa_write_summaries_of_cgraph_node_set (set);
> +	  ipa_write_optimization_summaries (set);
>  
>  	  lto_set_current_out_file (NULL);
>  	  lto_elf_file_close (file);
> @@ -1821,10 +1821,18 @@ read_cgraph_and_symbols (unsigned nfiles
>        gcc_assert (num_objects == nfiles);
>      }
>  
> +  if (!quiet_flag)
> +    fprintf (stderr, "Reading object files:");
> +
>    /* Read all of the object files specified on the command line.  */
>    for (i = 0, last_file_ix = 0; i < nfiles; ++i)
>      {
>        struct lto_file_decl_data *file_data = NULL;
> +      if (!quiet_flag)
> +	{
> +	  fprintf (stderr, " %s", fnames[i]);
> +	  fflush (stderr);
> +	}
>  
>        current_lto_file = lto_elf_file_open (fnames[i], false);
>        if (!current_lto_file)
> @@ -1851,9 +1859,15 @@ read_cgraph_and_symbols (unsigned nfiles
>    /* Each pass will set the appropriate timer.  */
>    timevar_pop (TV_IPA_LTO_DECL_IO);
>  
> +  if (!quiet_flag)
> +    fprintf (stderr, "\nReading the callgraph\n");
> +
>    /* Read the callgraph.  */
>    input_cgraph ();
>  
> +  if (!quiet_flag)
> +    fprintf (stderr, "Merging declarations\n");
> +
>    /* Merge global decls.  */
>    lto_symtab_merge_decls ();
>  
> @@ -1861,8 +1875,14 @@ read_cgraph_and_symbols (unsigned nfiles
>    lto_fixup_decls (all_file_decl_data);
>    free_gimple_type_tables ();
>  
> +  if (!quiet_flag)
> +    fprintf (stderr, "Reading summaries\n");
> +
>    /* Read the IPA summary data.  */
> -  ipa_read_summaries ();
> +  if (flag_ltrans)
> +    ipa_read_optimization_summaries ();
> +  else
> +    ipa_read_summaries ();
>  
>    /* Finally merge the cgraph according to the decl merging decisions.  */
>    lto_symtab_merge_cgraph_nodes ();
> @@ -1908,6 +1928,11 @@ materialize_cgraph (void)
>    unsigned i;
>    timevar_id_t lto_timer;
>  
> +  if (!quiet_flag)
> +    fprintf (stderr,
> +	     flag_wpa ? "Materializing decls:" : "Reading function bodies:");
> +
> +
>    /* Now that we have input the cgraph, we need to clear all of the aux
>       nodes and read the functions if we are not running in WPA mode.  */
>    timevar_push (TV_IPA_LTO_GIMPLE_IO);
> @@ -1926,6 +1951,7 @@ materialize_cgraph (void)
>        if (node->local.lto_file_data
>            && !DECL_IS_BUILTIN (node->decl))
>  	{
> +	  announce_function (node->decl);
>  	  lto_materialize_function (node);
>  	  lto_stats.num_input_cgraph_nodes++;
>  	}
> @@ -1949,6 +1975,8 @@ materialize_cgraph (void)
>  
>    /* Fix up any calls to DECLs that have become not exception throwing.  */
>    lto_fixup_nothrow_decls ();
> +  if (!quiet_flag)
> +    fprintf (stderr, "\n");
>  
>    timevar_pop (lto_timer);
>  }
> @@ -1986,7 +2014,14 @@ do_whole_program_analysis (void)
>  
>    lto_1_to_1_map ();
>  
> +  if (!quiet_flag)
> +    {
> +      fprintf (stderr, "\nStreaming out");
> +      fflush (stderr);
> +    }
>    output_files = lto_wpa_write_files ();
> +  if (!quiet_flag)
> +    fprintf (stderr, "\n");
>  
>    /* Show the LTO report before launching LTRANS.  */
>    if (flag_lto_report)
> Index: passes.c
> ===================================================================
> --- passes.c	(revision 158563)
> +++ passes.c	(working copy)
> @@ -1674,8 +1674,8 @@ ipa_write_summaries_1 (cgraph_node_set s
>    struct lto_out_decl_state *state = lto_new_out_decl_state ();
>    lto_push_out_decl_state (state);
>  
> -  if (!flag_wpa)
> -    ipa_write_summaries_2 (all_regular_ipa_passes, set, state);
> +  gcc_assert (!flag_wpa);
> +  ipa_write_summaries_2 (all_regular_ipa_passes, set, state);
>    ipa_write_summaries_2 (all_lto_gen_passes, set, state);
>  
>    gcc_assert (lto_get_out_decl_state () == state);
> @@ -1730,15 +1730,58 @@ ipa_write_summaries (void)
>    ggc_free (set);
>  }
>  
> +/* Same as execute_pass_list but assume that subpasses of IPA passes
> +   are local passes. If SET is not NULL, write out optimization summaries of
> +   only those node in SET. */
> +
> +static void
> +ipa_write_optimization_summaries_1 (struct opt_pass *pass, cgraph_node_set set,
> +		       struct lto_out_decl_state *state)
> +{
> +  while (pass)
> +    {
> +      struct ipa_opt_pass_d *ipa_pass = (struct ipa_opt_pass_d *)pass;
> +      gcc_assert (!current_function_decl);
> +      gcc_assert (!cfun);
> +      gcc_assert (pass->type == SIMPLE_IPA_PASS || pass->type == IPA_PASS);
> +      if (pass->type == IPA_PASS
> +	  && ipa_pass->write_optimization_summary
> +	  && (!pass->gate || pass->gate ()))
> +	{
> +	  /* If a timevar is present, start it.  */
> +	  if (pass->tv_id)
> +	    timevar_push (pass->tv_id);
> +
> +	  ipa_pass->write_optimization_summary (set);
> +
> +	  /* If a timevar is present, start it.  */
> +	  if (pass->tv_id)
> +	    timevar_pop (pass->tv_id);
> +	}
> +
> +      if (pass->sub && pass->sub->type != GIMPLE_PASS)
> +	ipa_write_optimization_summaries_1 (pass->sub, set, state);
> +
> +      pass = pass->next;
> +    }
> +}
>  
> -/* Write all the summaries for the cgraph nodes in SET.  If SET is
> +/* Write all the optimization summaries for the cgraph nodes in SET.  If SET is
>     NULL, write out all summaries of all nodes. */
>  
>  void
> -ipa_write_summaries_of_cgraph_node_set (cgraph_node_set set)
> +ipa_write_optimization_summaries (cgraph_node_set set)
>  {
> -  if (flag_generate_lto && !(errorcount || sorrycount))
> -    ipa_write_summaries_1 (set);
> +  struct lto_out_decl_state *state = lto_new_out_decl_state ();
> +  lto_push_out_decl_state (state);
> +
> +  gcc_assert (flag_wpa);
> +  ipa_write_optimization_summaries_1 (all_regular_ipa_passes, set, state);
> +  ipa_write_optimization_summaries_1 (all_lto_gen_passes, set, state);
> +
> +  gcc_assert (lto_get_out_decl_state () == state);
> +  lto_pop_out_decl_state ();
> +  lto_delete_out_decl_state (state);
>  }
>  
>  /* Same as execute_pass_list but assume that subpasses of IPA passes
> @@ -1783,13 +1826,57 @@ ipa_read_summaries_1 (struct opt_pass *p
>  void
>  ipa_read_summaries (void)
>  {
> -  if (!flag_ltrans)
> -    ipa_read_summaries_1 (all_regular_ipa_passes);
> +  ipa_read_summaries_1 (all_regular_ipa_passes);
>    ipa_read_summaries_1 (all_lto_gen_passes);
>  }
>  
>  /* Same as execute_pass_list but assume that subpasses of IPA passes
>     are local passes.  */
> +
> +static void
> +ipa_read_optimization_summaries_1 (struct opt_pass *pass)
> +{
> +  while (pass)
> +    {
> +      struct ipa_opt_pass_d *ipa_pass = (struct ipa_opt_pass_d *) pass;
> +
> +      gcc_assert (!current_function_decl);
> +      gcc_assert (!cfun);
> +      gcc_assert (pass->type == SIMPLE_IPA_PASS || pass->type == IPA_PASS);
> +
> +      if (pass->gate == NULL || pass->gate ())
> +	{
> +	  if (pass->type == IPA_PASS && ipa_pass->read_optimization_summary)
> +	    {
> +	      /* If a timevar is present, start it.  */
> +	      if (pass->tv_id)
> +		timevar_push (pass->tv_id);
> +
> +	      ipa_pass->read_optimization_summary ();
> +
> +	      /* Stop timevar.  */
> +	      if (pass->tv_id)
> +		timevar_pop (pass->tv_id);
> +	    }
> +
> +	  if (pass->sub && pass->sub->type != GIMPLE_PASS)
> +	    ipa_read_optimization_summaries_1 (pass->sub);
> +	}
> +      pass = pass->next;
> +    }
> +}
> +
> +/* Read all the summaries for all_regular_ipa_passes and all_lto_gen_passes.  */
> +
> +void
> +ipa_read_optimization_summaries (void)
> +{
> +  ipa_read_optimization_summaries_1 (all_regular_ipa_passes);
> +  ipa_read_optimization_summaries_1 (all_lto_gen_passes);
> +}
> +
> +/* Same as execute_pass_list but assume that subpasses of IPA passes
> +   are local passes.  */
>  void
>  execute_ipa_pass_list (struct opt_pass *pass)
>  {
> 
> 

-- 
Richard Guenther <rguenther@suse.de>
Novell / SUSE Labs
SUSE LINUX Products GmbH - Nuernberg - AG Nuernberg - HRB 16746 - GF: Markus Rex