Stream stmt histograms for LTO
Jan Hubicka
hubicka@ucw.cz
Thu Mar 28 15:31:00 GMT 2013
Hi,
this patch adds streaming of histograms that did not make it into LTO
previously. This restores some optimizations, such as specialization of
memcpy/memset inline code for expected size&alignment of the block.
I also added specialzed streaming functions for gcov counters. They may be
wider than HWI in some configurations so we want to be sure there are no
uncontrolled overflows and we can do bit of sanity check that the values are
not negative. (and probably skip streaming when profile is absent)
lto-profilebootstrapped/regtested x86_64-linux.
Will commit it shortly.
* data-streamer-in.c (streamer_read_gcov_count): New function.
* gimple-streamer-out.c: Include value-prof.h.
(output_gimple_stmt): Output histogram.
(output_bb): Use streamer_write_gcov_count.
* value-prof.c: Include data-streamer.h
(dump_histogram_value): Add HIST_TYPE_MAX.
(stream_out_histogram_value): New function.
(stream_in_histogram_value): New function.
* value-prof.h (enum hist_type): Add HIST_TYPE_MAX.
(stream_out_histogram_value, stream_in_histogram_value): Declare.
* data-streamer-out.c (streamer_write_gcov_count): New function.
(streamer_write_gcov_count_stream): New function.
* lto-cgraph.c (lto_output_edge): Update counter streaming.
(lto_output_node): Likewise.
(input_node, input_edge): Likewise.
* lto-streamer-out.c (output_cfg): Update streaming.
* lto-streamer-in.c (input_cfg): Likewise.
* data-streamer.h (streamer_write_gcov_count,
streamer_write_gcov_count_stream, streamer_read_gcov_count): Declare.
* gimple-streamer-in.c: Include value-prof.h
(input_gimple_stmt): Input histograms.
(input_bb): Update profile streaming.
Index: data-streamer-in.c
===================================================================
*** data-streamer-in.c (revision 197125)
--- data-streamer-in.c (working copy)
*************** streamer_read_hwi (struct lto_input_bloc
*** 158,160 ****
--- 158,170 ----
}
}
}
+
+ /* Read gcov_type value from IB. */
+
+ gcov_type
+ streamer_read_gcov_count (struct lto_input_block *ib)
+ {
+ gcov_type ret = streamer_read_hwi (ib);
+ gcc_assert (ret >= 0);
+ return ret;
+ }
Index: gimple-streamer-out.c
===================================================================
*** gimple-streamer-out.c (revision 197125)
--- gimple-streamer-out.c (working copy)
*************** along with GCC; see the file COPYING3.
*** 28,33 ****
--- 28,34 ----
#include "gimple-streamer.h"
#include "lto-streamer.h"
#include "tree-streamer.h"
+ #include "value-prof.h"
/* Output PHI function PHI to the main stream in OB. */
*************** output_gimple_stmt (struct output_block
*** 59,64 ****
--- 60,66 ----
enum gimple_code code;
enum LTO_tags tag;
struct bitpack_d bp;
+ histogram_value hist;
/* Emit identifying tag. */
code = gimple_code (stmt);
*************** output_gimple_stmt (struct output_block
*** 72,77 ****
--- 74,81 ----
if (is_gimple_assign (stmt))
bp_pack_value (&bp, gimple_assign_nontemporal_move_p (stmt), 1);
bp_pack_value (&bp, gimple_has_volatile_ops (stmt), 1);
+ hist = gimple_histogram_value (cfun, stmt);
+ bp_pack_value (&bp, hist != NULL, 1);
bp_pack_var_len_unsigned (&bp, stmt->gsbase.subcode);
/* Emit location information for the statement. */
*************** output_gimple_stmt (struct output_block
*** 167,172 ****
--- 171,178 ----
default:
gcc_unreachable ();
}
+ if (hist)
+ stream_out_histogram_value (ob, hist);
}
*************** output_bb (struct output_block *ob, basi
*** 183,189 ****
: LTO_bb0);
streamer_write_uhwi (ob, bb->index);
! streamer_write_hwi (ob, bb->count);
streamer_write_hwi (ob, bb->frequency);
streamer_write_hwi (ob, bb->flags);
--- 189,195 ----
: LTO_bb0);
streamer_write_uhwi (ob, bb->index);
! streamer_write_gcov_count (ob, bb->count);
streamer_write_hwi (ob, bb->frequency);
streamer_write_hwi (ob, bb->flags);
Index: value-prof.c
===================================================================
*** value-prof.c (revision 197125)
--- value-prof.c (working copy)
*************** along with GCC; see the file COPYING3.
*** 44,49 ****
--- 44,50 ----
#include "dumpfile.h"
#include "pointer-set.h"
#include "profile.h"
+ #include "data-streamer.h"
/* In this file value profile based optimizations are placed. Currently the
following optimizations are implemented (for more detailed descriptions
*************** dump_histogram_value (FILE *dump_file, h
*** 333,341 ****
--- 334,428 ----
}
fprintf (dump_file, ".\n");
break;
+ case HIST_TYPE_MAX:
+ gcc_unreachable ();
}
}
+ /* Dump information about HIST to DUMP_FILE. */
+
+ void
+ stream_out_histogram_value (struct output_block *ob, histogram_value hist)
+ {
+ struct bitpack_d bp;
+ unsigned int i;
+
+ bp = bitpack_create (ob->main_stream);
+ bp_pack_enum (&bp, hist_type, HIST_TYPE_MAX, hist->type);
+ bp_pack_value (&bp, hist->hvalue.next != NULL, 1);
+ streamer_write_bitpack (&bp);
+ switch (hist->type)
+ {
+ case HIST_TYPE_INTERVAL:
+ streamer_write_hwi (ob, hist->hdata.intvl.int_start);
+ streamer_write_uhwi (ob, hist->hdata.intvl.steps);
+ break;
+ default:
+ break;
+ }
+ for (i = 0; i < hist->n_counters; i++)
+ streamer_write_gcov_count (ob, hist->hvalue.counters[i]);
+ if (hist->hvalue.next)
+ stream_out_histogram_value (ob, hist->hvalue.next);
+ }
+ /* Dump information about HIST to DUMP_FILE. */
+
+ void
+ stream_in_histogram_value (struct lto_input_block *ib, gimple stmt)
+ {
+ enum hist_type type;
+ unsigned int ncounters = 0;
+ struct bitpack_d bp;
+ unsigned int i;
+ histogram_value new_val;
+ bool next;
+ histogram_value *next_p = NULL;
+
+ do
+ {
+ bp = streamer_read_bitpack (ib);
+ type = bp_unpack_enum (&bp, hist_type, HIST_TYPE_MAX);
+ next = bp_unpack_value (&bp, 1);
+ new_val = gimple_alloc_histogram_value (cfun, type, stmt, NULL);
+ switch (type)
+ {
+ case HIST_TYPE_INTERVAL:
+ new_val->hdata.intvl.int_start = streamer_read_hwi (ib);
+ ncounters = new_val->hdata.intvl.steps = streamer_read_uhwi (ib);
+ ncounters++;
+ break;
+
+ case HIST_TYPE_POW2:
+ case HIST_TYPE_AVERAGE:
+ ncounters = 2;
+ break;
+
+ case HIST_TYPE_SINGLE_VALUE:
+ case HIST_TYPE_CONST_DELTA:
+ case HIST_TYPE_INDIR_CALL:
+ ncounters = 3;
+ break;
+
+ case HIST_TYPE_IOR:
+ ncounters = 1;
+ break;
+ case HIST_TYPE_MAX:
+ gcc_unreachable ();
+ }
+ new_val->hvalue.counters = XNEWVAR (gcov_type, sizeof (*new_val->hvalue.counters) * ncounters);
+ new_val->n_counters = ncounters;
+ for (i = 0; i < ncounters; i++)
+ new_val->hvalue.counters[i] = streamer_read_gcov_count (ib);
+ debug_gimple_stmt (stmt);
+ if (!next_p)
+ gimple_add_histogram_value (cfun, stmt, new_val);
+ else
+ *next_p = new_val;
+ next_p = &new_val->hvalue.next;
+ }
+ while (next);
+ }
+
/* Dump all histograms attached to STMT to DUMP_FILE. */
void
Index: value-prof.h
===================================================================
*** value-prof.h (revision 197125)
--- value-prof.h (working copy)
*************** enum hist_type
*** 33,39 ****
HIST_TYPE_INDIR_CALL, /* Tries to identify the function that is (almost)
called in indirect call */
HIST_TYPE_AVERAGE, /* Compute average value (sum of all values). */
! HIST_TYPE_IOR /* Used to compute expected alignment. */
};
#define COUNTER_FOR_HIST_TYPE(TYPE) ((int) (TYPE) + GCOV_FIRST_VALUE_COUNTER)
--- 33,40 ----
HIST_TYPE_INDIR_CALL, /* Tries to identify the function that is (almost)
called in indirect call */
HIST_TYPE_AVERAGE, /* Compute average value (sum of all values). */
! HIST_TYPE_IOR, /* Used to compute expected alignment. */
! HIST_TYPE_MAX
};
#define COUNTER_FOR_HIST_TYPE(TYPE) ((int) (TYPE) + GCOV_FIRST_VALUE_COUNTER)
*************** extern void gimple_gen_const_delta_profi
*** 98,103 ****
--- 99,106 ----
unsigned, unsigned);
extern void gimple_gen_average_profiler (histogram_value, unsigned, unsigned);
extern void gimple_gen_ior_profiler (histogram_value, unsigned, unsigned);
+ extern void stream_out_histogram_value (struct output_block *, histogram_value);
+ extern void stream_in_histogram_value (struct lto_input_block *, gimple);
/* In profile.c. */
extern void init_branch_prob (void);
Index: data-streamer-out.c
===================================================================
*** data-streamer-out.c (revision 197125)
--- data-streamer-out.c (working copy)
*************** streamer_write_hwi (struct output_block
*** 174,179 ****
--- 174,186 ----
streamer_write_hwi_stream (ob->main_stream, work);
}
+ /* Write a gcov counter value WORK to OB->main_stream. */
+
+ void
+ streamer_write_gcov_count (struct output_block *ob, gcov_type work)
+ {
+ streamer_write_gcov_count_stream (ob->main_stream, work);
+ }
/* Write an unsigned HOST_WIDE_INT value WORK to OBS. */
*************** streamer_write_hwi_stream (struct lto_ou
*** 216,218 ****
--- 223,235 ----
}
while (more);
}
+
+ /* Write a GCOV counter value WORK to OBS. */
+
+ void
+ streamer_write_gcov_count_stream (struct lto_output_stream *obs, gcov_type work)
+ {
+ gcc_assert (work >= 0);
+ gcc_assert ((HOST_WIDE_INT) work == work);
+ streamer_write_hwi_stream (obs, work);
+ }
Index: lto-cgraph.c
===================================================================
*** lto-cgraph.c (revision 197125)
--- lto-cgraph.c (working copy)
*************** lto_output_edge (struct lto_simple_outpu
*** 267,273 ****
streamer_write_hwi_stream (ob->main_stream, ref);
}
! streamer_write_hwi_stream (ob->main_stream, edge->count);
bp = bitpack_create (ob->main_stream);
uid = (!gimple_has_body_p (edge->caller->symbol.decl)
--- 267,273 ----
streamer_write_hwi_stream (ob->main_stream, ref);
}
! streamer_write_gcov_count_stream (ob->main_stream, edge->count);
bp = bitpack_create (ob->main_stream);
uid = (!gimple_has_body_p (edge->caller->symbol.decl)
*************** lto_output_node (struct lto_simple_outpu
*** 429,435 ****
lto_output_fn_decl_index (ob->decl_state, ob->main_stream, node->symbol.decl);
! streamer_write_hwi_stream (ob->main_stream, node->count);
streamer_write_hwi_stream (ob->main_stream, node->count_materialization_scale);
streamer_write_hwi_stream (ob->main_stream,
--- 429,435 ----
lto_output_fn_decl_index (ob->decl_state, ob->main_stream, node->symbol.decl);
! streamer_write_gcov_count_stream (ob->main_stream, node->count);
streamer_write_hwi_stream (ob->main_stream, node->count_materialization_scale);
streamer_write_hwi_stream (ob->main_stream,
*************** input_node (struct lto_file_decl_data *f
*** 948,954 ****
if (order >= symtab_order)
symtab_order = order + 1;
! node->count = streamer_read_hwi (ib);
node->count_materialization_scale = streamer_read_hwi (ib);
count = streamer_read_hwi (ib);
--- 948,954 ----
if (order >= symtab_order)
symtab_order = order + 1;
! node->count = streamer_read_gcov_count (ib);
node->count_materialization_scale = streamer_read_hwi (ib);
count = streamer_read_hwi (ib);
*************** input_edge (struct lto_input_block *ib,
*** 1109,1115 ****
else
callee = NULL;
! count = (gcov_type) streamer_read_hwi (ib);
bp = streamer_read_bitpack (ib);
inline_failed = bp_unpack_enum (&bp, cgraph_inline_failed_enum, CIF_N_REASONS);
--- 1109,1115 ----
else
callee = NULL;
! count = streamer_read_gcov_count (ib);
bp = streamer_read_bitpack (ib);
inline_failed = bp_unpack_enum (&bp, cgraph_inline_failed_enum, CIF_N_REASONS);
Index: lto-streamer-out.c
===================================================================
*** lto-streamer-out.c (revision 197125)
--- lto-streamer-out.c (working copy)
*************** output_cfg (struct output_block *ob, str
*** 644,650 ****
{
streamer_write_uhwi (ob, e->dest->index);
streamer_write_hwi (ob, e->probability);
! streamer_write_hwi (ob, e->count);
streamer_write_uhwi (ob, e->flags);
}
}
--- 644,650 ----
{
streamer_write_uhwi (ob, e->dest->index);
streamer_write_hwi (ob, e->probability);
! streamer_write_gcov_count (ob, e->count);
streamer_write_uhwi (ob, e->flags);
}
}
Index: lto-streamer-in.c
===================================================================
*** lto-streamer-in.c (revision 197125)
--- lto-streamer-in.c (working copy)
*************** input_cfg (struct lto_input_block *ib, s
*** 622,628 ****
dest_index = streamer_read_uhwi (ib);
probability = (int) streamer_read_hwi (ib);
! count = ((gcov_type) streamer_read_hwi (ib) * count_materialization_scale
+ REG_BR_PROB_BASE / 2) / REG_BR_PROB_BASE;
edge_flags = streamer_read_uhwi (ib);
--- 622,628 ----
dest_index = streamer_read_uhwi (ib);
probability = (int) streamer_read_hwi (ib);
! count = ((gcov_type) streamer_read_gcov_count (ib) * count_materialization_scale
+ REG_BR_PROB_BASE / 2) / REG_BR_PROB_BASE;
edge_flags = streamer_read_uhwi (ib);
Index: data-streamer.h
===================================================================
*** data-streamer.h (revision 197125)
--- data-streamer.h (working copy)
*************** HOST_WIDE_INT bp_unpack_var_len_int (str
*** 63,68 ****
--- 63,69 ----
void streamer_write_zero (struct output_block *);
void streamer_write_uhwi (struct output_block *, unsigned HOST_WIDE_INT);
void streamer_write_hwi (struct output_block *, HOST_WIDE_INT);
+ void streamer_write_gcov_count (struct output_block *, gcov_type);
void streamer_write_string (struct output_block *, struct lto_output_stream *,
const char *, bool);
unsigned streamer_string_index (struct output_block *, const char *,
*************** void bp_pack_string (struct output_block
*** 77,82 ****
--- 78,84 ----
void streamer_write_uhwi_stream (struct lto_output_stream *,
unsigned HOST_WIDE_INT);
void streamer_write_hwi_stream (struct lto_output_stream *, HOST_WIDE_INT);
+ void streamer_write_gcov_count_stream (struct lto_output_stream *, gcov_type);
/* In data-streamer-in.c */
const char *string_for_index (struct data_in *, unsigned int, unsigned int *);
*************** const char *bp_unpack_indexed_string (st
*** 89,94 ****
--- 91,97 ----
const char *bp_unpack_string (struct data_in *, struct bitpack_d *);
unsigned HOST_WIDE_INT streamer_read_uhwi (struct lto_input_block *);
HOST_WIDE_INT streamer_read_hwi (struct lto_input_block *);
+ gcov_type streamer_read_gcov_count (struct lto_input_block *);
/* Returns a hash code for P. Adapted from libiberty's htab_hash_string
to support strings that may not end in '\0'. */
Index: gimple-streamer-in.c
===================================================================
*** gimple-streamer-in.c (revision 197125)
--- gimple-streamer-in.c (working copy)
*************** along with GCC; see the file COPYING3.
*** 28,33 ****
--- 28,34 ----
#include "data-streamer.h"
#include "tree-streamer.h"
#include "gimple-streamer.h"
+ #include "value-prof.h"
/* Read a PHI function for basic block BB in function FN. DATA_IN is
the file being read. IB is the input block to use for reading. */
*************** input_gimple_stmt (struct lto_input_bloc
*** 86,91 ****
--- 87,93 ----
unsigned HOST_WIDE_INT num_ops;
size_t i;
struct bitpack_d bp;
+ bool has_hist;
code = lto_tag_to_gimple_code (tag);
*************** input_gimple_stmt (struct lto_input_bloc
*** 97,102 ****
--- 99,105 ----
if (is_gimple_assign (stmt))
stmt->gsbase.nontemporal_move = bp_unpack_value (&bp, 1);
stmt->gsbase.has_volatile_ops = bp_unpack_value (&bp, 1);
+ has_hist = bp_unpack_value (&bp, 1);
stmt->gsbase.subcode = bp_unpack_var_len_unsigned (&bp);
/* Read location information. */
*************** input_gimple_stmt (struct lto_input_bloc
*** 300,305 ****
--- 303,310 ----
/* Mark the statement modified so its operand vectors can be filled in. */
gimple_set_modified (stmt, true);
+ if (has_hist)
+ stream_in_histogram_value (ib, stmt);
return stmt;
}
*************** input_bb (struct lto_input_block *ib, en
*** 324,330 ****
index = streamer_read_uhwi (ib);
bb = BASIC_BLOCK_FOR_FUNCTION (fn, index);
! bb->count = (streamer_read_hwi (ib) * count_materialization_scale
+ REG_BR_PROB_BASE / 2) / REG_BR_PROB_BASE;
bb->frequency = streamer_read_hwi (ib);
bb->flags = streamer_read_hwi (ib);
--- 329,335 ----
index = streamer_read_uhwi (ib);
bb = BASIC_BLOCK_FOR_FUNCTION (fn, index);
! bb->count = (streamer_read_gcov_count (ib) * count_materialization_scale
+ REG_BR_PROB_BASE / 2) / REG_BR_PROB_BASE;
bb->frequency = streamer_read_hwi (ib);
bb->flags = streamer_read_hwi (ib);
Index: Makefile.in
===================================================================
*** Makefile.in (revision 197125)
--- Makefile.in (working copy)
*************** lto-compress.o: lto-compress.c $(CONFIG_
*** 2137,2145 ****
$(TREE_H) langhooks.h $(LTO_STREAMER_H) $(LTO_SECTION_H) \
lto-compress.h $(DIAGNOSTIC_CORE_H) $(DIAGNOSTIC_CORE_H)
data-streamer-in.o: data-streamer-in.c $(CONFIG_H) $(SYSTEM_H) coretypes.h \
! $(DATA_STREAMER_H) $(DIAGNOSTIC_H)
data-streamer-out.o: data-streamer-out.c $(CONFIG_H) $(SYSTEM_H) coretypes.h \
! $(DATA_STREAMER_H)
data-streamer.o: data-streamer.c $(CONFIG_H) $(SYSTEM_H) coretypes.h \
$(DATA_STREAMER_H)
gimple-streamer-in.o: gimple-streamer-in.c $(CONFIG_H) $(SYSTEM_H) \
--- 2137,2145 ----
$(TREE_H) langhooks.h $(LTO_STREAMER_H) $(LTO_SECTION_H) \
lto-compress.h $(DIAGNOSTIC_CORE_H) $(DIAGNOSTIC_CORE_H)
data-streamer-in.o: data-streamer-in.c $(CONFIG_H) $(SYSTEM_H) coretypes.h \
! $(DATA_STREAMER_H) $(DIAGNOSTIC_H) value-prof.h
data-streamer-out.o: data-streamer-out.c $(CONFIG_H) $(SYSTEM_H) coretypes.h \
! $(DATA_STREAMER_H) value-prof.h
data-streamer.o: data-streamer.c $(CONFIG_H) $(SYSTEM_H) coretypes.h \
$(DATA_STREAMER_H)
gimple-streamer-in.o: gimple-streamer-in.c $(CONFIG_H) $(SYSTEM_H) \
*************** value-prof.o : value-prof.c $(CONFIG_H)
*** 3088,3094 ****
$(RECOG_H) insn-config.h $(OPTABS_H) $(REGS_H) $(GGC_H) $(DIAGNOSTIC_H) \
$(TREE_H) $(COVERAGE_H) $(RTL_H) $(GCOV_IO_H) $(TREE_FLOW_H) \
tree-flow-inline.h $(TIMEVAR_H) $(DIAGNOSTIC_CORE_H) pointer-set.h \
! $(GIMPLE_PRETTY_PRINT_H)
loop-doloop.o : loop-doloop.c $(CONFIG_H) $(SYSTEM_H) coretypes.h dumpfile.h $(TM_H) \
$(RTL_H) $(FLAGS_H) $(EXPR_H) hard-reg-set.h $(BASIC_BLOCK_H) $(TM_P_H) \
$(DIAGNOSTIC_CORE_H) $(CFGLOOP_H) $(PARAMS_H) $(TARGET_H)
--- 3088,3094 ----
$(RECOG_H) insn-config.h $(OPTABS_H) $(REGS_H) $(GGC_H) $(DIAGNOSTIC_H) \
$(TREE_H) $(COVERAGE_H) $(RTL_H) $(GCOV_IO_H) $(TREE_FLOW_H) \
tree-flow-inline.h $(TIMEVAR_H) $(DIAGNOSTIC_CORE_H) pointer-set.h \
! $(GIMPLE_PRETTY_PRINT_H) $(DATA_STREAMER_H)
loop-doloop.o : loop-doloop.c $(CONFIG_H) $(SYSTEM_H) coretypes.h dumpfile.h $(TM_H) \
$(RTL_H) $(FLAGS_H) $(EXPR_H) hard-reg-set.h $(BASIC_BLOCK_H) $(TM_P_H) \
$(DIAGNOSTIC_CORE_H) $(CFGLOOP_H) $(PARAMS_H) $(TARGET_H)
More information about the Gcc-patches
mailing list