This is the mail archive of the
gcc-patches@gcc.gnu.org
mailing list for the GCC project.
Put inliners info about cgraph edges into its own summary structure
- From: Jan Hubicka <hubicka at ucw dot cz>
- To: gcc-patches at gcc dot gnu dot org
- Date: Tue, 26 Apr 2011 19:35:52 +0200
- Subject: Put inliners info about cgraph edges into its own summary structure
Hi,
this patch moves inliner related callgraph edge info
(call_stmt_size/time and loop_nest) into inliner own edge summaries,
just like we have for ipa-prop.
Primary motivation for doing so is that I don't want to add edge predicates
to main cgraph edge and handle them in lto-cgraph, but doing this change
also turned out to be quite nice cleanup. In particular the loop nest
handling was quite spread across the compiler. Now all updating happens
at one spot in ipa-inline-analysis.
The patch also fixes several bugs with loop nest updating: we never updated
loop nests of indirect edges when inlining and also some code assumed that no
nest is 0, other 1. This lead to underflows in the nest counter.
The patch also adds more human readable dump of inline decisions than
the callgraph with inlined_to pointers.
Bootstrapped/regtested x86_64-linux, will commit it later today.
Honza
* cgraphbuild.c (build_cgraph_edges): Update call
of cgraph_create_edge and cgraph_create_indirect_edge.
* cgraph.c (cgraph_create_edge_including_clones,
cgraph_create_edge_1, cgraph_allocate_init_indirect_info,
cgraph_update_edges_for_call_stmt_node): Do not take nest
argument; do not initialize call_stmt_size/time.
(dump_cgraph_node): Do not dump nest.
(cgraph_clone_edge): Do not take loop_nest argument;
do not propagate it; do not clone call_stmt_size/time.
(cgraph_clone_node): Likewise.
(cgraph_create_virtual_clone): Update.
* cgraph.h (struct cgraph_edge): Remove
call_stmt_size/call_stmt_time/loop_nest.
(cgraph_create_edge, cgraph_create_indirect_edge,
cgraph_create_edge_including_clones, cgraph_clone_node): Update
prototype.
* tree-emutls.c (gen_emutls_addr): Update.
* ipa-inline-transform.c (update_noncloned_frequencies): Do not handle
loop_nest; handle indirect calls, too.
(clone_inlined_nodes): Do not care about updating inline summaries.
* cgraphunit.c (cgraph_copy_node_for_versioning): Update.
* lto-cgraph.c (lto_output_edge, input_node, input_edge): Do not
stream call_stmt_size/call_stmt_time/loop_nest.
* ipa-inline.c (edge_badness): Update.
(ipa_inline): dump summaries after inlining.
* ipa-inline.h (struct inline_edge_summary, inline_edge_summary_t):
new.
(inline_edge_summary): New function.
* ipa-inline-analysis.c (edge_duplication_hook_holder): New holder.
(inline_edge_removal_hook): Handle edge summaries.
(inline_edge_duplication_hook): New hook.
(inline_summary_alloc): Alloc hooks.
(initialize_growth_caches): Do not register removal hooks.
(free_growth_caches); Do not free removal hook.
(dump_inline_edge_summary): New function.
(dump_inline_summary): Use it.
(estimate_function_body_sizes, estimate_edge_size_and_time): Update.
(inline_update_callee_summaries): New function.
(inline_merge_summary): Use it.
(do_estimate_edge_time, do_estimate_edge_growth): Update.
(read_inline_edge_summary): New function.
(inline_read_section): Use it.
(write_inline_edge_summary): New function.
(inline_write_summary): Use it.
(inline_free_summary): Free edge new holders.
* tree-inline.c (copy_bb): Update.
Index: cgraphbuild.c
===================================================================
*** cgraphbuild.c (revision 172962)
--- cgraphbuild.c (working copy)
*************** build_cgraph_edges (void)
*** 354,365 ****
decl = gimple_call_fndecl (stmt);
if (decl)
cgraph_create_edge (node, cgraph_get_create_node (decl),
! stmt, bb->count, freq, bb->loop_depth);
else
cgraph_create_indirect_edge (node, stmt,
gimple_call_flags (stmt),
! bb->count, freq,
! bb->loop_depth);
}
walk_stmt_load_store_addr_ops (stmt, node, mark_load,
mark_store, mark_address);
--- 354,364 ----
decl = gimple_call_fndecl (stmt);
if (decl)
cgraph_create_edge (node, cgraph_get_create_node (decl),
! stmt, bb->count, freq);
else
cgraph_create_indirect_edge (node, stmt,
gimple_call_flags (stmt),
! bb->count, freq);
}
walk_stmt_load_store_addr_ops (stmt, node, mark_load,
mark_store, mark_address);
*************** rebuild_cgraph_edges (void)
*** 464,475 ****
decl = gimple_call_fndecl (stmt);
if (decl)
cgraph_create_edge (node, cgraph_get_create_node (decl), stmt,
! bb->count, freq, bb->loop_depth);
else
cgraph_create_indirect_edge (node, stmt,
gimple_call_flags (stmt),
! bb->count, freq,
! bb->loop_depth);
}
walk_stmt_load_store_addr_ops (stmt, node, mark_load,
mark_store, mark_address);
--- 463,473 ----
decl = gimple_call_fndecl (stmt);
if (decl)
cgraph_create_edge (node, cgraph_get_create_node (decl), stmt,
! bb->count, freq);
else
cgraph_create_indirect_edge (node, stmt,
gimple_call_flags (stmt),
! bb->count, freq);
}
walk_stmt_load_store_addr_ops (stmt, node, mark_load,
mark_store, mark_address);
Index: cgraph.c
===================================================================
*** cgraph.c (revision 172962)
--- cgraph.c (working copy)
*************** cgraph_create_edge_including_clones (str
*** 921,927 ****
struct cgraph_node *callee,
gimple old_stmt,
gimple stmt, gcov_type count,
! int freq, int loop_depth,
cgraph_inline_failed_t reason)
{
struct cgraph_node *node;
--- 921,927 ----
struct cgraph_node *callee,
gimple old_stmt,
gimple stmt, gcov_type count,
! int freq,
cgraph_inline_failed_t reason)
{
struct cgraph_node *node;
*************** cgraph_create_edge_including_clones (str
*** 929,935 ****
if (!cgraph_edge (orig, stmt))
{
! edge = cgraph_create_edge (orig, callee, stmt, count, freq, loop_depth);
edge->inline_failed = reason;
}
--- 929,935 ----
if (!cgraph_edge (orig, stmt))
{
! edge = cgraph_create_edge (orig, callee, stmt, count, freq);
edge->inline_failed = reason;
}
*************** cgraph_create_edge_including_clones (str
*** 948,954 ****
else if (!cgraph_edge (node, stmt))
{
edge = cgraph_create_edge (node, callee, stmt, count,
! freq, loop_depth);
edge->inline_failed = reason;
}
--- 948,954 ----
else if (!cgraph_edge (node, stmt))
{
edge = cgraph_create_edge (node, callee, stmt, count,
! freq);
edge->inline_failed = reason;
}
*************** cgraph_create_edge_including_clones (str
*** 972,978 ****
static struct cgraph_edge *
cgraph_create_edge_1 (struct cgraph_node *caller, struct cgraph_node *callee,
! gimple call_stmt, gcov_type count, int freq, int nest)
{
struct cgraph_edge *edge;
--- 972,978 ----
static struct cgraph_edge *
cgraph_create_edge_1 (struct cgraph_node *caller, struct cgraph_node *callee,
! gimple call_stmt, gcov_type count, int freq)
{
struct cgraph_edge *edge;
*************** cgraph_create_edge_1 (struct cgraph_node
*** 1011,1021 ****
edge->frequency = freq;
gcc_assert (freq >= 0);
gcc_assert (freq <= CGRAPH_FREQ_MAX);
- edge->loop_nest = nest;
edge->call_stmt = call_stmt;
- edge->call_stmt_size = 0;
- edge->call_stmt_time = 0;
push_cfun (DECL_STRUCT_FUNCTION (caller->decl));
edge->can_throw_external
= call_stmt ? stmt_can_throw_external (call_stmt) : false;
--- 1011,1018 ----
*************** cgraph_create_edge_1 (struct cgraph_node
*** 1035,1044 ****
struct cgraph_edge *
cgraph_create_edge (struct cgraph_node *caller, struct cgraph_node *callee,
! gimple call_stmt, gcov_type count, int freq, int nest)
{
struct cgraph_edge *edge = cgraph_create_edge_1 (caller, callee, call_stmt,
! count, freq, nest);
edge->indirect_unknown_callee = 0;
initialize_inline_failed (edge);
--- 1032,1041 ----
struct cgraph_edge *
cgraph_create_edge (struct cgraph_node *caller, struct cgraph_node *callee,
! gimple call_stmt, gcov_type count, int freq)
{
struct cgraph_edge *edge = cgraph_create_edge_1 (caller, callee, call_stmt,
! count, freq);
edge->indirect_unknown_callee = 0;
initialize_inline_failed (edge);
*************** cgraph_allocate_init_indirect_info (void
*** 1074,1083 ****
struct cgraph_edge *
cgraph_create_indirect_edge (struct cgraph_node *caller, gimple call_stmt,
int ecf_flags,
! gcov_type count, int freq, int nest)
{
struct cgraph_edge *edge = cgraph_create_edge_1 (caller, NULL, call_stmt,
! count, freq, nest);
edge->indirect_unknown_callee = 1;
initialize_inline_failed (edge);
--- 1071,1080 ----
struct cgraph_edge *
cgraph_create_indirect_edge (struct cgraph_node *caller, gimple call_stmt,
int ecf_flags,
! gcov_type count, int freq)
{
struct cgraph_edge *edge = cgraph_create_edge_1 (caller, NULL, call_stmt,
! count, freq);
edge->indirect_unknown_callee = 1;
initialize_inline_failed (edge);
*************** cgraph_update_edges_for_call_stmt_node (
*** 1244,1250 ****
struct cgraph_edge *ne = NULL;
gcov_type count;
int frequency;
- int loop_nest;
if (e)
{
--- 1241,1246 ----
*************** cgraph_update_edges_for_call_stmt_node (
*** 1268,1274 ****
attached to edge is invalid. */
count = e->count;
frequency = e->frequency;
- loop_nest = e->loop_nest;
cgraph_remove_edge (e);
}
else
--- 1264,1269 ----
*************** cgraph_update_edges_for_call_stmt_node (
*** 1278,1291 ****
count = bb->count;
frequency = compute_call_stmt_bb_frequency (current_function_decl,
bb);
- loop_nest = bb->loop_depth;
}
if (new_call)
{
ne = cgraph_create_edge (node, cgraph_get_create_node (new_call),
! new_stmt, count, frequency,
! loop_nest);
gcc_assert (ne->inline_failed);
}
}
--- 1273,1284 ----
count = bb->count;
frequency = compute_call_stmt_bb_frequency (current_function_decl,
bb);
}
if (new_call)
{
ne = cgraph_create_edge (node, cgraph_get_create_node (new_call),
! new_stmt, count, frequency);
gcc_assert (ne->inline_failed);
}
}
*************** dump_cgraph_node (FILE *f, struct cgraph
*** 1915,1922 ****
if (edge->frequency)
fprintf (f, "(%.2f per call) ",
edge->frequency / (double)CGRAPH_FREQ_BASE);
- if (edge->loop_nest)
- fprintf (f, "(nested in %i loops) ", edge->loop_nest);
if (edge->can_throw_external)
fprintf(f, "(can throw external) ");
}
--- 1908,1913 ----
*************** cgraph_function_possibly_inlined_p (tree
*** 2064,2070 ****
struct cgraph_edge *
cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n,
gimple call_stmt, unsigned stmt_uid, gcov_type count_scale,
! int freq_scale, int loop_nest, bool update_original)
{
struct cgraph_edge *new_edge;
gcov_type count = e->count * count_scale / REG_BR_PROB_BASE;
--- 2055,2061 ----
struct cgraph_edge *
cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n,
gimple call_stmt, unsigned stmt_uid, gcov_type count_scale,
! int freq_scale, bool update_original)
{
struct cgraph_edge *new_edge;
gcov_type count = e->count * count_scale / REG_BR_PROB_BASE;
*************** cgraph_clone_edge (struct cgraph_edge *e
*** 2085,2106 ****
{
struct cgraph_node *callee = cgraph_get_node (decl);
gcc_checking_assert (callee);
! new_edge = cgraph_create_edge (n, callee, call_stmt, count, freq,
! e->loop_nest + loop_nest);
}
else
{
new_edge = cgraph_create_indirect_edge (n, call_stmt,
e->indirect_info->ecf_flags,
! count, freq,
! e->loop_nest + loop_nest);
*new_edge->indirect_info = *e->indirect_info;
}
}
else
{
! new_edge = cgraph_create_edge (n, e->callee, call_stmt, count, freq,
! e->loop_nest + loop_nest);
if (e->indirect_info)
{
new_edge->indirect_info
--- 2076,2094 ----
{
struct cgraph_node *callee = cgraph_get_node (decl);
gcc_checking_assert (callee);
! new_edge = cgraph_create_edge (n, callee, call_stmt, count, freq);
}
else
{
new_edge = cgraph_create_indirect_edge (n, call_stmt,
e->indirect_info->ecf_flags,
! count, freq);
*new_edge->indirect_info = *e->indirect_info;
}
}
else
{
! new_edge = cgraph_create_edge (n, e->callee, call_stmt, count, freq);
if (e->indirect_info)
{
new_edge->indirect_info
*************** cgraph_clone_edge (struct cgraph_edge *e
*** 2109,2116 ****
}
}
- new_edge->call_stmt_size = e->call_stmt_size;
- new_edge->call_stmt_time = e->call_stmt_time;
new_edge->inline_failed = e->inline_failed;
new_edge->indirect_inlining_edge = e->indirect_inlining_edge;
new_edge->lto_stmt_uid = stmt_uid;
--- 2097,2102 ----
*************** cgraph_clone_edge (struct cgraph_edge *e
*** 2137,2143 ****
by node. */
struct cgraph_node *
cgraph_clone_node (struct cgraph_node *n, tree decl, gcov_type count, int freq,
! int loop_nest, bool update_original,
VEC(cgraph_edge_p,heap) *redirect_callers)
{
struct cgraph_node *new_node = cgraph_create_node_1 ();
--- 2123,2129 ----
by node. */
struct cgraph_node *
cgraph_clone_node (struct cgraph_node *n, tree decl, gcov_type count, int freq,
! bool update_original,
VEC(cgraph_edge_p,heap) *redirect_callers)
{
struct cgraph_node *new_node = cgraph_create_node_1 ();
*************** cgraph_clone_node (struct cgraph_node *n
*** 2188,2198 ****
for (e = n->callees;e; e=e->next_callee)
cgraph_clone_edge (e, new_node, e->call_stmt, e->lto_stmt_uid,
! count_scale, freq, loop_nest, update_original);
for (e = n->indirect_calls; e; e = e->next_callee)
cgraph_clone_edge (e, new_node, e->call_stmt, e->lto_stmt_uid,
! count_scale, freq, loop_nest, update_original);
ipa_clone_references (new_node, NULL, &n->ref_list);
new_node->next_sibling_clone = n->clones;
--- 2174,2184 ----
for (e = n->callees;e; e=e->next_callee)
cgraph_clone_edge (e, new_node, e->call_stmt, e->lto_stmt_uid,
! count_scale, freq, update_original);
for (e = n->indirect_calls; e; e = e->next_callee)
cgraph_clone_edge (e, new_node, e->call_stmt, e->lto_stmt_uid,
! count_scale, freq, update_original);
ipa_clone_references (new_node, NULL, &n->ref_list);
new_node->next_sibling_clone = n->clones;
*************** cgraph_create_virtual_clone (struct cgra
*** 2285,2291 ****
SET_DECL_RTL (new_decl, NULL);
new_node = cgraph_clone_node (old_node, new_decl, old_node->count,
! CGRAPH_FREQ_BASE, 0, false,
redirect_callers);
/* Update the properties.
Make clone visible only within this translation unit. Make sure
--- 2271,2277 ----
SET_DECL_RTL (new_decl, NULL);
new_node = cgraph_clone_node (old_node, new_decl, old_node->count,
! CGRAPH_FREQ_BASE, false,
redirect_callers);
/* Update the properties.
Make clone visible only within this translation unit. Make sure
*************** cgraph_propagate_frequency (struct cgrap
*** 2700,2706 ****
fprintf (dump_file, " Called by %s that is executed once\n",
cgraph_node_name (edge->caller));
maybe_unlikely_executed = false;
! if (edge->loop_nest)
{
maybe_executed_once = false;
if (dump_file && (dump_flags & TDF_DETAILS))
--- 2686,2692 ----
fprintf (dump_file, " Called by %s that is executed once\n",
cgraph_node_name (edge->caller));
maybe_unlikely_executed = false;
! if (inline_edge_summary (edge)->loop_depth)
{
maybe_executed_once = false;
if (dump_file && (dump_flags & TDF_DETAILS))
Index: cgraph.h
===================================================================
*** cgraph.h (revision 172962)
--- cgraph.h (working copy)
*************** struct GTY((chain_next ("%h.next_caller"
*** 381,391 ****
int frequency;
/* Unique id of the edge. */
int uid;
- /* Estimated size and time of the call statement. */
- int call_stmt_size;
- int call_stmt_time;
- /* Depth of loop nest, 1 means no loop nest. */
- unsigned short int loop_nest;
/* Whether this edge was made direct by indirect inlining. */
unsigned int indirect_inlining_edge : 1;
/* Whether this edge describes an indirect call with an undetermined
--- 381,386 ----
*************** void cgraph_release_function_body (struc
*** 504,512 ****
void cgraph_node_remove_callees (struct cgraph_node *node);
struct cgraph_edge *cgraph_create_edge (struct cgraph_node *,
struct cgraph_node *,
! gimple, gcov_type, int, int);
struct cgraph_edge *cgraph_create_indirect_edge (struct cgraph_node *, gimple,
! int, gcov_type, int, int);
struct cgraph_indirect_call_info *cgraph_allocate_init_indirect_info (void);
struct cgraph_node * cgraph_get_node (const_tree);
struct cgraph_node * cgraph_get_node_or_alias (const_tree);
--- 499,507 ----
void cgraph_node_remove_callees (struct cgraph_node *node);
struct cgraph_edge *cgraph_create_edge (struct cgraph_node *,
struct cgraph_node *,
! gimple, gcov_type, int);
struct cgraph_edge *cgraph_create_indirect_edge (struct cgraph_node *, gimple,
! int, gcov_type, int);
struct cgraph_indirect_call_info *cgraph_allocate_init_indirect_info (void);
struct cgraph_node * cgraph_get_node (const_tree);
struct cgraph_node * cgraph_get_node_or_alias (const_tree);
*************** void cgraph_set_call_stmt (struct cgraph
*** 522,528 ****
void cgraph_set_call_stmt_including_clones (struct cgraph_node *, gimple, gimple);
void cgraph_create_edge_including_clones (struct cgraph_node *,
struct cgraph_node *,
! gimple, gimple, gcov_type, int, int,
cgraph_inline_failed_t);
void cgraph_update_edges_for_call_stmt (gimple, tree, gimple);
struct cgraph_local_info *cgraph_local_info (tree);
--- 517,523 ----
void cgraph_set_call_stmt_including_clones (struct cgraph_node *, gimple, gimple);
void cgraph_create_edge_including_clones (struct cgraph_node *,
struct cgraph_node *,
! gimple, gimple, gcov_type, int,
cgraph_inline_failed_t);
void cgraph_update_edges_for_call_stmt (gimple, tree, gimple);
struct cgraph_local_info *cgraph_local_info (tree);
*************** struct cgraph_rtl_info *cgraph_rtl_info
*** 531,538 ****
const char * cgraph_node_name (struct cgraph_node *);
struct cgraph_edge * cgraph_clone_edge (struct cgraph_edge *,
struct cgraph_node *, gimple,
! unsigned, gcov_type, int, int, bool);
! struct cgraph_node * cgraph_clone_node (struct cgraph_node *, tree, gcov_type, int,
int, bool, VEC(cgraph_edge_p,heap) *);
void cgraph_redirect_edge_callee (struct cgraph_edge *, struct cgraph_node *);
--- 526,533 ----
const char * cgraph_node_name (struct cgraph_node *);
struct cgraph_edge * cgraph_clone_edge (struct cgraph_edge *,
struct cgraph_node *, gimple,
! unsigned, gcov_type, int, bool);
! struct cgraph_node * cgraph_clone_node (struct cgraph_node *, tree, gcov_type,
int, bool, VEC(cgraph_edge_p,heap) *);
void cgraph_redirect_edge_callee (struct cgraph_edge *, struct cgraph_node *);
Index: tree-emutls.c
===================================================================
*** tree-emutls.c (revision 172962)
--- tree-emutls.c (working copy)
*************** gen_emutls_addr (tree decl, struct lower
*** 437,443 ****
gimple_seq_add_stmt (&d->seq, x);
cgraph_create_edge (d->cfun_node, d->builtin_node, x,
! d->bb->count, d->bb_freq, d->bb->loop_depth);
/* We may be adding a new reference to a new variable to the function.
This means we have to play with the ipa-reference web. */
--- 437,443 ----
gimple_seq_add_stmt (&d->seq, x);
cgraph_create_edge (d->cfun_node, d->builtin_node, x,
! d->bb->count, d->bb_freq);
/* We may be adding a new reference to a new variable to the function.
This means we have to play with the ipa-reference web. */
Index: ipa-inline-transform.c
===================================================================
*** ipa-inline-transform.c (revision 172962)
--- ipa-inline-transform.c (working copy)
*************** along with GCC; see the file COPYING3.
*** 49,60 ****
int ncalls_inlined;
int nfunctions_inlined;
! /* Scale frequency of NODE edges by FREQ_SCALE and increase loop nest
! by NEST. */
static void
update_noncloned_frequencies (struct cgraph_node *node,
! int freq_scale, int nest)
{
struct cgraph_edge *e;
--- 49,59 ----
int ncalls_inlined;
int nfunctions_inlined;
! /* Scale frequency of NODE edges by FREQ_SCALE. */
static void
update_noncloned_frequencies (struct cgraph_node *node,
! int freq_scale)
{
struct cgraph_edge *e;
*************** update_noncloned_frequencies (struct cgr
*** 63,74 ****
freq_scale = 1;
for (e = node->callees; e; e = e->next_callee)
{
- e->loop_nest += nest;
e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
if (e->frequency > CGRAPH_FREQ_MAX)
e->frequency = CGRAPH_FREQ_MAX;
if (!e->inline_failed)
! update_noncloned_frequencies (e->callee, freq_scale, nest);
}
}
--- 62,78 ----
freq_scale = 1;
for (e = node->callees; e; e = e->next_callee)
{
e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
if (e->frequency > CGRAPH_FREQ_MAX)
e->frequency = CGRAPH_FREQ_MAX;
if (!e->inline_failed)
! update_noncloned_frequencies (e->callee, freq_scale);
! }
! for (e = node->indirect_calls; e; e = e->next_callee)
! {
! e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
! if (e->frequency > CGRAPH_FREQ_MAX)
! e->frequency = CGRAPH_FREQ_MAX;
}
}
*************** void
*** 83,91 ****
clone_inlined_nodes (struct cgraph_edge *e, bool duplicate,
bool update_original, int *overall_size)
{
- HOST_WIDE_INT peak;
- struct inline_summary *caller_info, *callee_info;
-
if (duplicate)
{
/* We may eliminate the need for out-of-line copy to be output.
--- 87,92 ----
*************** clone_inlined_nodes (struct cgraph_edge
*** 125,158 ****
}
duplicate = false;
e->callee->local.externally_visible = false;
! update_noncloned_frequencies (e->callee, e->frequency, e->loop_nest);
}
else
{
struct cgraph_node *n;
n = cgraph_clone_node (e->callee, e->callee->decl,
! e->count, e->frequency, e->loop_nest,
update_original, NULL);
cgraph_redirect_edge_callee (e, n);
}
}
- callee_info = inline_summary (e->callee);
- caller_info = inline_summary (e->caller);
-
if (e->caller->global.inlined_to)
e->callee->global.inlined_to = e->caller->global.inlined_to;
else
e->callee->global.inlined_to = e->caller;
- callee_info->stack_frame_offset
- = caller_info->stack_frame_offset
- + caller_info->estimated_self_stack_size;
- peak = callee_info->stack_frame_offset
- + callee_info->estimated_self_stack_size;
- if (inline_summary (e->callee->global.inlined_to)->estimated_stack_size
- < peak)
- inline_summary (e->callee->global.inlined_to)->estimated_stack_size = peak;
- cgraph_propagate_frequency (e->callee);
/* Recursively clone all bodies. */
for (e = e->callee->callees; e; e = e->next_callee)
--- 126,147 ----
}
duplicate = false;
e->callee->local.externally_visible = false;
! update_noncloned_frequencies (e->callee, e->frequency);
}
else
{
struct cgraph_node *n;
n = cgraph_clone_node (e->callee, e->callee->decl,
! e->count, e->frequency,
update_original, NULL);
cgraph_redirect_edge_callee (e, n);
}
}
if (e->caller->global.inlined_to)
e->callee->global.inlined_to = e->caller->global.inlined_to;
else
e->callee->global.inlined_to = e->caller;
/* Recursively clone all bodies. */
for (e = e->callee->callees; e; e = e->next_callee)
*************** inline_call (struct cgraph_edge *e, bool
*** 187,199 ****
to = e->caller;
if (to->global.inlined_to)
to = to->global.inlined_to;
- old_size = inline_summary (to)->size;
- inline_merge_summary (e);
- new_size = inline_summary (to)->size;
clone_inlined_nodes (e, true, update_original, overall_size);
gcc_assert (curr->callee->global.inlined_to == to);
if (overall_size && new_size > old_size)
*overall_size += new_size - old_size;
ncalls_inlined++;
--- 176,189 ----
to = e->caller;
if (to->global.inlined_to)
to = to->global.inlined_to;
clone_inlined_nodes (e, true, update_original, overall_size);
gcc_assert (curr->callee->global.inlined_to == to);
+
+ old_size = inline_summary (to)->size;
+ inline_merge_summary (e);
+ new_size = inline_summary (to)->size;
if (overall_size && new_size > old_size)
*overall_size += new_size - old_size;
ncalls_inlined++;
Index: cgraphunit.c
===================================================================
*** cgraphunit.c (revision 172962)
--- cgraphunit.c (working copy)
*************** cgraph_copy_node_for_versioning (struct
*** 2001,2014 ****
cgraph_clone_edge (e, new_version, e->call_stmt,
e->lto_stmt_uid, REG_BR_PROB_BASE,
CGRAPH_FREQ_BASE,
! e->loop_nest, true);
for (e = old_version->indirect_calls; e; e=e->next_callee)
if (!bbs_to_copy
|| bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index))
cgraph_clone_edge (e, new_version, e->call_stmt,
e->lto_stmt_uid, REG_BR_PROB_BASE,
CGRAPH_FREQ_BASE,
! e->loop_nest, true);
FOR_EACH_VEC_ELT (cgraph_edge_p, redirect_callers, i, e)
{
/* Redirect calls to the old version node to point to its new
--- 2001,2014 ----
cgraph_clone_edge (e, new_version, e->call_stmt,
e->lto_stmt_uid, REG_BR_PROB_BASE,
CGRAPH_FREQ_BASE,
! true);
for (e = old_version->indirect_calls; e; e=e->next_callee)
if (!bbs_to_copy
|| bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index))
cgraph_clone_edge (e, new_version, e->call_stmt,
e->lto_stmt_uid, REG_BR_PROB_BASE,
CGRAPH_FREQ_BASE,
! true);
FOR_EACH_VEC_ELT (cgraph_edge_p, redirect_callers, i, e)
{
/* Redirect calls to the old version node to point to its new
Index: lto-cgraph.c
===================================================================
*** lto-cgraph.c (revision 172962)
--- lto-cgraph.c (working copy)
*************** lto_output_edge (struct lto_simple_outpu
*** 285,293 ****
bp_pack_value (&bp, uid, HOST_BITS_PER_INT);
bp_pack_value (&bp, edge->inline_failed, HOST_BITS_PER_INT);
bp_pack_value (&bp, edge->frequency, HOST_BITS_PER_INT);
- bp_pack_value (&bp, edge->call_stmt_size, HOST_BITS_PER_INT);
- bp_pack_value (&bp, edge->call_stmt_time, HOST_BITS_PER_INT);
- bp_pack_value (&bp, edge->loop_nest, 30);
bp_pack_value (&bp, edge->indirect_inlining_edge, 1);
bp_pack_value (&bp, edge->call_stmt_cannot_inline_p, 1);
bp_pack_value (&bp, edge->can_throw_external, 1);
--- 285,290 ----
*************** input_node (struct lto_file_decl_data *f
*** 1003,1009 ****
if (clone_ref != LCC_NOT_FOUND)
{
node = cgraph_clone_node (VEC_index (cgraph_node_ptr, nodes, clone_ref), fn_decl,
! 0, CGRAPH_FREQ_BASE, 0, false, NULL);
}
else
node = cgraph_get_create_node (fn_decl);
--- 1000,1006 ----
if (clone_ref != LCC_NOT_FOUND)
{
node = cgraph_clone_node (VEC_index (cgraph_node_ptr, nodes, clone_ref), fn_decl,
! 0, CGRAPH_FREQ_BASE, false, NULL);
}
else
node = cgraph_get_create_node (fn_decl);
*************** input_edge (struct lto_input_block *ib,
*** 1164,1174 ****
unsigned int stmt_id;
gcov_type count;
int freq;
- unsigned int nest;
cgraph_inline_failed_t inline_failed;
struct bitpack_d bp;
int ecf_flags = 0;
- int call_stmt_time, call_stmt_size;
caller = VEC_index (cgraph_node_ptr, nodes, lto_input_sleb128 (ib));
if (caller == NULL || caller->decl == NULL_TREE)
--- 1161,1169 ----
*************** input_edge (struct lto_input_block *ib,
*** 1190,1211 ****
inline_failed = (cgraph_inline_failed_t) bp_unpack_value (&bp,
HOST_BITS_PER_INT);
freq = (int) bp_unpack_value (&bp, HOST_BITS_PER_INT);
- call_stmt_size = (int) bp_unpack_value (&bp, HOST_BITS_PER_INT);
- call_stmt_time = (int) bp_unpack_value (&bp, HOST_BITS_PER_INT);
- nest = (unsigned) bp_unpack_value (&bp, 30);
if (indirect)
! edge = cgraph_create_indirect_edge (caller, NULL, 0, count, freq, nest);
else
! edge = cgraph_create_edge (caller, callee, NULL, count, freq, nest);
edge->indirect_inlining_edge = bp_unpack_value (&bp, 1);
edge->lto_stmt_uid = stmt_id;
edge->inline_failed = inline_failed;
edge->call_stmt_cannot_inline_p = bp_unpack_value (&bp, 1);
edge->can_throw_external = bp_unpack_value (&bp, 1);
- edge->call_stmt_size = call_stmt_size;
- edge->call_stmt_time = call_stmt_time;
if (indirect)
{
if (bp_unpack_value (&bp, 1))
--- 1185,1201 ----
inline_failed = (cgraph_inline_failed_t) bp_unpack_value (&bp,
HOST_BITS_PER_INT);
freq = (int) bp_unpack_value (&bp, HOST_BITS_PER_INT);
if (indirect)
! edge = cgraph_create_indirect_edge (caller, NULL, 0, count, freq);
else
! edge = cgraph_create_edge (caller, callee, NULL, count, freq);
edge->indirect_inlining_edge = bp_unpack_value (&bp, 1);
edge->lto_stmt_uid = stmt_id;
edge->inline_failed = inline_failed;
edge->call_stmt_cannot_inline_p = bp_unpack_value (&bp, 1);
edge->can_throw_external = bp_unpack_value (&bp, 1);
if (indirect)
{
if (bp_unpack_value (&bp, 1))
Index: ipa-inline.c
===================================================================
*** ipa-inline.c (revision 172962)
--- ipa-inline.c (working copy)
*************** edge_badness (struct cgraph_edge *edge,
*** 739,745 ****
of functions fully inlined in program. */
else
{
! int nest = MIN (edge->loop_nest, 8);
badness = estimate_growth (edge->callee) * 256;
/* Decrease badness if call is nested. */
--- 739,745 ----
of functions fully inlined in program. */
else
{
! int nest = MIN (inline_edge_summary (edge)->loop_depth, 8);
badness = estimate_growth (edge->callee) * 256;
/* Decrease badness if call is nested. */
*************** recursive_inlining (struct cgraph_edge *
*** 1027,1033 ****
{
/* We need original clone to copy around. */
master_clone = cgraph_clone_node (node, node->decl,
! node->count, CGRAPH_FREQ_BASE, 1,
false, NULL);
for (e = master_clone->callees; e; e = e->next_callee)
if (!e->inline_failed)
--- 1027,1033 ----
{
/* We need original clone to copy around. */
master_clone = cgraph_clone_node (node, node->decl,
! node->count, CGRAPH_FREQ_BASE,
false, NULL);
for (e = master_clone->callees; e; e = e->next_callee)
if (!e->inline_failed)
*************** ipa_inline (void)
*** 1555,1560 ****
--- 1555,1562 ----
"\nInlined %i calls, eliminated %i functions\n\n",
ncalls_inlined, nfunctions_inlined);
+ if (dump_file)
+ dump_inline_summaries (dump_file);
/* In WPA we use inline summaries for partitioning process. */
if (!flag_wpa)
inline_free_summary ();
*************** early_inliner (void)
*** 1709,1717 ****
info that might be cleared out for newly discovered edges. */
for (edge = node->callees; edge; edge = edge->next_callee)
{
! edge->call_stmt_size
= estimate_num_insns (edge->call_stmt, &eni_size_weights);
! edge->call_stmt_time
= estimate_num_insns (edge->call_stmt, &eni_time_weights);
}
timevar_pop (TV_INTEGRATION);
--- 1711,1720 ----
info that might be cleared out for newly discovered edges. */
for (edge = node->callees; edge; edge = edge->next_callee)
{
! struct inline_edge_summary *es = inline_edge_summary (edge);
! es->call_stmt_size
= estimate_num_insns (edge->call_stmt, &eni_size_weights);
! es->call_stmt_time
= estimate_num_insns (edge->call_stmt, &eni_time_weights);
}
timevar_pop (TV_INTEGRATION);
Index: ipa-inline.h
===================================================================
*** ipa-inline.h (revision 172962)
--- ipa-inline.h (working copy)
*************** struct GTY(()) inline_summary
*** 101,106 ****
--- 101,108 ----
int time;
int size;
+ /* Conditional size/time information. The summaries are being
+ merged during inlining. */
conditions conds;
VEC(size_time_entry,gc) *entry;
};
*************** DEF_VEC_O(inline_summary_t);
*** 110,115 ****
--- 112,132 ----
DEF_VEC_ALLOC_O(inline_summary_t,gc);
extern GTY(()) VEC(inline_summary_t,gc) *inline_summary_vec;
+ /* Information kept about callgraph edges. */
+ struct inline_edge_summary
+ {
+ /* Estimated size and time of the call statement. */
+ int call_stmt_size;
+ int call_stmt_time;
+ /* Depth of loop nest, 0 means no nesting. */
+ unsigned short int loop_depth;
+ };
+
+ typedef struct inline_edge_summary inline_edge_summary_t;
+ DEF_VEC_O(inline_edge_summary_t);
+ DEF_VEC_ALLOC_O(inline_edge_summary_t,heap);
+ extern VEC(inline_edge_summary_t,heap) *inline_edge_summary_vec;
+
typedef struct edge_growth_cache_entry
{
int time, size;
*************** inline_summary (struct cgraph_node *node
*** 152,157 ****
--- 169,180 ----
return VEC_index (inline_summary_t, inline_summary_vec, node->uid);
}
+ static inline struct inline_edge_summary *
+ inline_edge_summary (struct cgraph_edge *edge)
+ {
+ return VEC_index (inline_edge_summary_t,
+ inline_edge_summary_vec, edge->uid);
+ }
/* Return estimated unit growth after inlning all calls to NODE.
Quick accesors to the inline growth caches.
Index: ipa-inline-analysis.c
===================================================================
*** ipa-inline-analysis.c (revision 172962)
--- ipa-inline-analysis.c (working copy)
*************** enum predicate_conditions
*** 110,123 ****
--- 110,129 ----
static struct cgraph_node_hook_list *function_insertion_hook_holder;
static struct cgraph_node_hook_list *node_removal_hook_holder;
static struct cgraph_2node_hook_list *node_duplication_hook_holder;
+ static struct cgraph_2edge_hook_list *edge_duplication_hook_holder;
static struct cgraph_edge_hook_list *edge_removal_hook_holder;
static void inline_node_removal_hook (struct cgraph_node *, void *);
static void inline_node_duplication_hook (struct cgraph_node *,
struct cgraph_node *, void *);
+ static void inline_edge_removal_hook (struct cgraph_edge *, void *);
+ static void inline_edge_duplication_hook (struct cgraph_edge *,
+ struct cgraph_edge *,
+ void *);
/* VECtor holding inline summaries.
In GGC memory because conditions might point to constant trees. */
VEC(inline_summary_t,gc) *inline_summary_vec;
+ VEC(inline_edge_summary_t,heap) *inline_edge_summary_vec;
/* Cached node/edge growths. */
VEC(int,heap) *node_growth_cache;
*************** inline_summary_alloc (void)
*** 512,525 ****
--- 518,541 ----
if (!node_removal_hook_holder)
node_removal_hook_holder =
cgraph_add_node_removal_hook (&inline_node_removal_hook, NULL);
+ if (!edge_removal_hook_holder)
+ edge_removal_hook_holder =
+ cgraph_add_edge_removal_hook (&inline_edge_removal_hook, NULL);
if (!node_duplication_hook_holder)
node_duplication_hook_holder =
cgraph_add_node_duplication_hook (&inline_node_duplication_hook, NULL);
+ if (!edge_duplication_hook_holder)
+ edge_duplication_hook_holder =
+ cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook, NULL);
if (VEC_length (inline_summary_t, inline_summary_vec)
<= (unsigned) cgraph_max_uid)
VEC_safe_grow_cleared (inline_summary_t, gc,
inline_summary_vec, cgraph_max_uid + 1);
+ if (VEC_length (inline_edge_summary_t, inline_edge_summary_vec)
+ <= (unsigned) cgraph_edge_max_uid)
+ VEC_safe_grow_cleared (inline_edge_summary_t, heap,
+ inline_edge_summary_vec, cgraph_edge_max_uid + 1);
}
/* Hook that is called by cgraph.c when a node is removed. */
*************** inline_node_removal_hook (struct cgraph_
*** 540,545 ****
--- 556,562 ----
memset (info, 0, sizeof (inline_summary_t));
}
+
/* Hook that is called by cgraph.c when a node is duplicated. */
static void
*************** inline_node_duplication_hook (struct cgr
*** 556,567 ****
}
/* Keep edge cache consistent across edge removal. */
static void
inline_edge_removal_hook (struct cgraph_edge *edge, void *data ATTRIBUTE_UNUSED)
{
! reset_edge_growth_cache (edge);
}
--- 573,601 ----
}
+ /* Hook that is called by cgraph.c when a node is duplicated. */
+
+ static void
+ inline_edge_duplication_hook (struct cgraph_edge *src, struct cgraph_edge *dst,
+ ATTRIBUTE_UNUSED void *data)
+ {
+ struct inline_edge_summary *info;
+ inline_summary_alloc ();
+ info = inline_edge_summary (dst);
+ memcpy (info, inline_edge_summary (src),
+ sizeof (struct inline_edge_summary));
+ }
+
+
/* Keep edge cache consistent across edge removal. */
static void
inline_edge_removal_hook (struct cgraph_edge *edge, void *data ATTRIBUTE_UNUSED)
{
! if (edge_growth_cache)
! reset_edge_growth_cache (edge);
! if (edge->uid < (int)VEC_length (inline_edge_summary_t, inline_edge_summary_vec))
! memset (inline_edge_summary (edge), 0, sizeof (struct inline_edge_summary));
}
*************** inline_edge_removal_hook (struct cgraph_
*** 570,578 ****
void
initialize_growth_caches (void)
{
- if (!edge_removal_hook_holder)
- edge_removal_hook_holder =
- cgraph_add_edge_removal_hook (&inline_edge_removal_hook, NULL);
if (cgraph_edge_max_uid)
VEC_safe_grow_cleared (edge_growth_cache_entry, heap, edge_growth_cache,
cgraph_edge_max_uid);
--- 604,609 ----
*************** initialize_growth_caches (void)
*** 586,593 ****
void
free_growth_caches (void)
{
- if (edge_removal_hook_holder)
- cgraph_remove_edge_removal_hook (edge_removal_hook_holder);
VEC_free (edge_growth_cache_entry, heap, edge_growth_cache);
edge_growth_cache = 0;
VEC_free (int, heap, node_growth_cache);
--- 617,622 ----
*************** free_growth_caches (void)
*** 595,600 ****
--- 624,665 ----
}
+ /* Dump edge summaries associated to NODE and recursively to all clones.
+ Indent by INDENT. */
+
+ static void
+ dump_inline_edge_summary (FILE * f, int indent, struct cgraph_node *node)
+ {
+ struct cgraph_edge *edge;
+ for (edge = node->callees; edge; edge = edge->next_callee)
+ {
+ struct inline_edge_summary *es = inline_edge_summary (edge);
+ fprintf (f, "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i time: %2i\n",
+ indent, "", cgraph_node_name (edge->callee),
+ edge->callee->uid,
+ edge->inline_failed ? "inlined"
+ : cgraph_inline_failed_string (edge->inline_failed),
+ indent, "",
+ es->loop_depth,
+ edge->frequency,
+ es->call_stmt_size,
+ es->call_stmt_time);
+ if (!edge->inline_failed)
+ dump_inline_edge_summary (f, indent+2, edge->callee);
+ }
+ for (edge = node->indirect_calls; edge; edge = edge->next_callee)
+ {
+ struct inline_edge_summary *es = inline_edge_summary (edge);
+ fprintf (f, "%*sindirect call loop depth:%2i freq:%4i size:%2i time: %2i\n",
+ indent, "",
+ es->loop_depth,
+ edge->frequency,
+ es->call_stmt_size,
+ es->call_stmt_time);
+ }
+ }
+
+
static void
dump_inline_summary (FILE * f, struct cgraph_node *node)
{
*************** dump_inline_summary (FILE * f, struct cg
*** 630,635 ****
--- 695,702 ----
(double) e->time / INLINE_TIME_SCALE);
dump_predicate (f, s->conds, &e->predicate);
}
+ fprintf (f, " calls:\n");
+ dump_inline_edge_summary (f, 4, node);
fprintf (f, "\n");
}
}
*************** dump_inline_summaries (FILE *f)
*** 646,652 ****
struct cgraph_node *node;
for (node = cgraph_nodes; node; node = node->next)
! if (node->analyzed)
dump_inline_summary (f, node);
}
--- 713,719 ----
struct cgraph_node *node;
for (node = cgraph_nodes; node; node = node->next)
! if (node->analyzed && !node->global.inlined_to)
dump_inline_summary (f, node);
}
*************** estimate_function_body_sizes (struct cgr
*** 919,926 ****
if (is_gimple_call (stmt))
{
struct cgraph_edge *edge = cgraph_edge (node, stmt);
! edge->call_stmt_size = this_size;
! edge->call_stmt_time = this_time;
/* Do not inline calls where we cannot triviall work around
mismatches in argument or return types. */
--- 986,996 ----
if (is_gimple_call (stmt))
{
struct cgraph_edge *edge = cgraph_edge (node, stmt);
! struct inline_edge_summary *es = inline_edge_summary (edge);
!
! es->call_stmt_size = this_size;
! es->call_stmt_time = this_time;
! es->loop_depth = bb->loop_depth;
/* Do not inline calls where we cannot triviall work around
mismatches in argument or return types. */
*************** struct gimple_opt_pass pass_inline_param
*** 1076,1083 ****
static void
estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *time)
{
! *size += e->call_stmt_size * INLINE_SIZE_SCALE;
! *time += (e->call_stmt_time
* e->frequency * (INLINE_TIME_SCALE / CGRAPH_FREQ_BASE));
if (*time > MAX_TIME * INLINE_TIME_SCALE)
*time = MAX_TIME * INLINE_TIME_SCALE;
--- 1146,1154 ----
static void
estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *time)
{
! struct inline_edge_summary *es = inline_edge_summary (e);
! *size += es->call_stmt_size * INLINE_SIZE_SCALE;
! *time += (es->call_stmt_time
* e->frequency * (INLINE_TIME_SCALE / CGRAPH_FREQ_BASE));
if (*time > MAX_TIME * INLINE_TIME_SCALE)
*time = MAX_TIME * INLINE_TIME_SCALE;
*************** remap_predicate (struct inline_summary *
*** 1222,1227 ****
--- 1293,1330 ----
}
+ /* Update summary information of inline clones after inlining.
+ Compute peak stack usage. */
+
+ static void
+ inline_update_callee_summaries (struct cgraph_node *node,
+ int depth)
+ {
+ struct cgraph_edge *e;
+ struct inline_summary *callee_info = inline_summary (node);
+ struct inline_summary *caller_info = inline_summary (node->callers->caller);
+ HOST_WIDE_INT peak;
+
+ callee_info->stack_frame_offset
+ = caller_info->stack_frame_offset
+ + caller_info->estimated_self_stack_size;
+ peak = callee_info->stack_frame_offset
+ + callee_info->estimated_self_stack_size;
+ if (inline_summary (node->global.inlined_to)->estimated_stack_size
+ < peak)
+ inline_summary (node->global.inlined_to)->estimated_stack_size = peak;
+ cgraph_propagate_frequency (node);
+ for (e = node->callees; e; e = e->next_callee)
+ {
+ if (!e->inline_failed)
+ inline_update_callee_summaries (e->callee, depth);
+ inline_edge_summary (e)->loop_depth += depth;
+ }
+ for (e = node->indirect_calls; e; e = e->next_callee)
+ inline_edge_summary (e)->loop_depth += depth;
+ }
+
+
/* We inlined EDGE. Update summary of the function we inlined into. */
void
*************** inline_merge_summary (struct cgraph_edge
*** 1275,1280 ****
--- 1378,1387 ----
for (i = 0; VEC_iterate (size_time_entry, info->entry, i, e); i++)
info->size += e->size, info->time += e->time;
estimate_calls_size_and_time (to, &info->size, &info->time);
+
+ inline_update_callee_summaries (edge->callee,
+ inline_edge_summary (edge)->loop_depth);
+
info->time = (info->time + INLINE_TIME_SCALE / 2) / INLINE_TIME_SCALE;
info->size = (info->size + INLINE_SIZE_SCALE / 2) / INLINE_SIZE_SCALE;
}
*************** do_estimate_edge_time (struct cgraph_edg
*** 1293,1303 ****
int time;
int size;
gcov_type ret;
gcc_checking_assert (edge->inline_failed);
estimate_callee_size_and_time (edge, true, &size, &time);
! ret = (((gcov_type)time - edge->call_stmt_time) * edge->frequency
+ CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE;
if (ret > MAX_TIME)
ret = MAX_TIME;
--- 1400,1411 ----
int time;
int size;
gcov_type ret;
+ struct inline_edge_summary *es = inline_edge_summary (edge);
gcc_checking_assert (edge->inline_failed);
estimate_callee_size_and_time (edge, true, &size, &time);
! ret = (((gcov_type)time - es->call_stmt_time) * edge->frequency
+ CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE;
if (ret > MAX_TIME)
ret = MAX_TIME;
*************** do_estimate_edge_time (struct cgraph_edg
*** 1313,1320 ****
VEC_index (edge_growth_cache_entry, edge_growth_cache, edge->uid)->time
= ret + (ret >= 0);
! ret_size = size - edge->call_stmt_size;
! gcc_checking_assert (edge->call_stmt_size);
VEC_index (edge_growth_cache_entry, edge_growth_cache, edge->uid)->size
= ret_size + (ret_size >= 0);
}
--- 1421,1428 ----
VEC_index (edge_growth_cache_entry, edge_growth_cache, edge->uid)->time
= ret + (ret >= 0);
! ret_size = size - es->call_stmt_size;
! gcc_checking_assert (es->call_stmt_size);
VEC_index (edge_growth_cache_entry, edge_growth_cache, edge->uid)->size
= ret_size + (ret_size >= 0);
}
*************** do_estimate_edge_growth (struct cgraph_e
*** 1345,1352 ****
/* Early inliner runs without caching, go ahead and do the dirty work. */
gcc_checking_assert (edge->inline_failed);
estimate_callee_size_and_time (edge, true, &size, NULL);
! gcc_checking_assert (edge->call_stmt_size);
! return size - edge->call_stmt_size;
}
--- 1453,1460 ----
/* Early inliner runs without caching, go ahead and do the dirty work. */
gcc_checking_assert (edge->inline_failed);
estimate_callee_size_and_time (edge, true, &size, NULL);
! gcc_checking_assert (inline_edge_summary (edge)->call_stmt_size);
! return size - inline_edge_summary (edge)->call_stmt_size;
}
*************** inline_generate_summary (void)
*** 1494,1499 ****
--- 1602,1619 ----
}
+ /* Write inline summary for edge E to OB. */
+
+ static void
+ read_inline_edge_summary (struct lto_input_block *ib, struct cgraph_edge *e)
+ {
+ struct inline_edge_summary *es = inline_edge_summary (e);
+ es->call_stmt_size = lto_input_uleb128 (ib);
+ es->call_stmt_time = lto_input_uleb128 (ib);
+ es->loop_depth = lto_input_uleb128 (ib);
+ }
+
+
/* Stream in inline summaries from the section. */
static void
*************** inline_read_section (struct lto_file_dec
*** 1524,1529 ****
--- 1644,1650 ----
struct inline_summary *info;
lto_cgraph_encoder_t encoder;
struct bitpack_d bp;
+ struct cgraph_edge *e;
index = lto_input_uleb128 (&ib);
encoder = file_data->cgraph_node_encoder;
*************** inline_read_section (struct lto_file_dec
*** 1568,1573 ****
--- 1689,1698 ----
VEC_safe_push (size_time_entry, gc, info->entry, &e);
}
+ for (e = node->callees; e; e = e->next_callee)
+ read_inline_edge_summary (&ib, e);
+ for (e = node->indirect_calls; e; e = e->next_callee)
+ read_inline_edge_summary (&ib, e);
}
lto_free_section_data (file_data, LTO_section_inline_summary, NULL, data,
*************** inline_read_summary (void)
*** 1611,1616 ****
--- 1736,1752 ----
cgraph_add_function_insertion_hook (&add_new_function, NULL);
}
+ /* Write inline summary for edge E to OB. */
+
+ static void
+ write_inline_edge_summary (struct output_block *ob, struct cgraph_edge *e)
+ {
+ struct inline_edge_summary *es = inline_edge_summary (e);
+ lto_output_uleb128_stream (ob->main_stream, es->call_stmt_size);
+ lto_output_uleb128_stream (ob->main_stream, es->call_stmt_time);
+ lto_output_uleb128_stream (ob->main_stream, es->loop_depth);
+ }
+
/* Write inline summary for node in SET.
Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
*************** inline_write_summary (cgraph_node_set se
*** 1638,1643 ****
--- 1774,1780 ----
{
struct inline_summary *info = inline_summary (node);
struct bitpack_d bp;
+ struct cgraph_edge *edge;
int i;
size_time_entry *e;
struct condition *c;
*************** inline_write_summary (cgraph_node_set se
*** 1684,1689 ****
--- 1821,1830 ----
}
lto_output_uleb128_stream (ob->main_stream, 0);
}
+ for (edge = node->callees; edge; edge = edge->next_callee)
+ write_inline_edge_summary (ob, edge);
+ for (edge = node->indirect_calls; edge; edge = edge->next_callee)
+ write_inline_edge_summary (ob, edge);
}
}
lto_output_1_stream (ob->main_stream, 0);
*************** inline_free_summary (void)
*** 1705,1713 ****
--- 1846,1858 ----
function_insertion_hook_holder = NULL;
if (node_removal_hook_holder)
cgraph_remove_node_removal_hook (node_removal_hook_holder);
+ if (edge_removal_hook_holder)
+ cgraph_remove_edge_removal_hook (edge_removal_hook_holder);
node_removal_hook_holder = NULL;
if (node_duplication_hook_holder)
cgraph_remove_node_duplication_hook (node_duplication_hook_holder);
+ if (edge_duplication_hook_holder)
+ cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder);
node_duplication_hook_holder = NULL;
VEC_free (inline_summary_t, gc, inline_summary_vec);
inline_summary_vec = NULL;
Index: tree-inline.c
===================================================================
*** tree-inline.c (revision 172962)
--- tree-inline.c (working copy)
*************** copy_bb (copy_body_data *id, basic_block
*** 1678,1684 ****
edge = cgraph_clone_edge (edge, id->dst_node, stmt,
gimple_uid (stmt),
REG_BR_PROB_BASE, CGRAPH_FREQ_BASE,
! edge->frequency, true);
/* We could also just rescale the frequency, but
doing so would introduce roundoff errors and make
verifier unhappy. */
--- 1678,1684 ----
edge = cgraph_clone_edge (edge, id->dst_node, stmt,
gimple_uid (stmt),
REG_BR_PROB_BASE, CGRAPH_FREQ_BASE,
! true);
/* We could also just rescale the frequency, but
doing so would introduce roundoff errors and make
verifier unhappy. */
*************** copy_bb (copy_body_data *id, basic_block
*** 1745,1757 ****
(id->dst_node, dest, orig_stmt, stmt, bb->count,
compute_call_stmt_bb_frequency (id->dst_node->decl,
copy_basic_block),
! bb->loop_depth, CIF_ORIGINALLY_INDIRECT_CALL);
else
cgraph_create_edge (id->dst_node, dest, stmt,
bb->count,
compute_call_stmt_bb_frequency
! (id->dst_node->decl, copy_basic_block),
! bb->loop_depth)->inline_failed
= CIF_ORIGINALLY_INDIRECT_CALL;
if (dump_file)
{
--- 1745,1756 ----
(id->dst_node, dest, orig_stmt, stmt, bb->count,
compute_call_stmt_bb_frequency (id->dst_node->decl,
copy_basic_block),
! CIF_ORIGINALLY_INDIRECT_CALL);
else
cgraph_create_edge (id->dst_node, dest, stmt,
bb->count,
compute_call_stmt_bb_frequency
! (id->dst_node->decl, copy_basic_block))->inline_failed
= CIF_ORIGINALLY_INDIRECT_CALL;
if (dump_file)
{