Index: gcc/cgraphbuild.c =================================================================== --- gcc/cgraphbuild.c (revision 201022) +++ gcc/cgraphbuild.c (working copy) @@ -37,7 +37,6 @@ along with GCC; see the file COPYING3. If not see #include "except.h" #include "l-ipo.h" #include "ipa-inline.h" -#include "auto-profile.h" /* Context of record_reference. */ struct record_reference_ctx @@ -491,9 +490,6 @@ build_cgraph_edges (void) tree decl; unsigned ix; - if (flag_auto_profile) - afdo_set_current_function_count (); - /* Create the callgraph edges and record the nodes referenced by the function. body. */ FOR_EACH_BB (bb) Index: gcc/Makefile.in =================================================================== --- gcc/Makefile.in (revision 201022) +++ gcc/Makefile.in (working copy) @@ -2880,8 +2880,7 @@ cgraphunit.o : cgraphunit.c $(CONFIG_H) $(SYSTEM_H $(FIBHEAP_H) output.h $(PARAMS_H) $(RTL_H) $(IPA_PROP_H) \ gt-cgraphunit.h tree-iterator.h $(COVERAGE_H) $(TREE_DUMP_H) \ $(GIMPLE_PRETTY_PRINT_H) $(IPA_INLINE_H) $(IPA_UTILS_H) l-ipo.h \ - $(LTO_STREAMER_H) output.h $(REGSET_H) $(EXCEPT_H) $(GCC_PLUGIN_H) plugin.h \ - $(AUTO_PROFILE_H) + $(LTO_STREAMER_H) output.h $(REGSET_H) $(EXCEPT_H) $(GCC_PLUGIN_H) plugin.h cgraphclones.o : cgraphclones.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \ $(TREE_H) langhooks.h $(TREE_INLINE_H) toplev.h $(DIAGNOSTIC_CORE_H) $(FLAGS_H) $(GGC_H) \ $(TARGET_H) $(CGRAPH_H) intl.h pointer-set.h $(FUNCTION_H) $(GIMPLE_H) \ @@ -2889,11 +2888,11 @@ cgraphclones.o : cgraphclones.c $(CONFIG_H) $(SYST $(PARAMS_H) $(RTL_H) $(IPA_PROP_H) \ tree-iterator.h $(COVERAGE_H) \ $(GIMPLE_PRETTY_PRINT_H) $(IPA_INLINE_H) $(IPA_UTILS_H) \ - $(LTO_STREAMER_H) $(EXCEPT_H) $(GCC_PLUGIN_H) $(AUTO_PROFILE_H) gt-cgraphclones.h + $(LTO_STREAMER_H) $(EXCEPT_H) $(GCC_PLUGIN_H) gt-cgraphclones.h cgraphbuild.o : cgraphbuild.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \ $(TREE_H) langhooks.h $(CGRAPH_H) intl.h pointer-set.h $(GIMPLE_H) \ $(TREE_FLOW_H) $(TREE_PASS_H) $(IPA_UTILS_H) $(EXCEPT_H) \ - $(IPA_INLINE_H) l-ipo.h $(AUTO_PROFILE_H) + $(IPA_INLINE_H) l-ipo.h varpool.o : varpool.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \ $(TREE_H) $(CGRAPH_H) langhooks.h $(DIAGNOSTIC_CORE_H) $(HASHTAB_H) \ $(GGC_H) $(TIMEVAR_H) debug.h $(TARGET_H) output.h $(GIMPLE_H) \ @@ -3390,7 +3389,7 @@ predict.o: predict.c $(CONFIG_H) $(SYSTEM_H) coret hard-reg-set.h $(DIAGNOSTIC_CORE_H) $(RECOG_H) $(FUNCTION_H) $(EXCEPT_H) \ $(TM_P_H) $(PREDICT_H) sreal.h $(PARAMS_H) $(TARGET_H) $(CFGLOOP_H) \ $(COVERAGE_H) $(SCEV_H) $(GGC_H) predict.def \ - $(TREE_FLOW_H) $(TREE_PASS_H) $(EXPR_H) pointer-set.h $(AUTO_PROFILE_H) + $(TREE_FLOW_H) $(TREE_PASS_H) $(EXPR_H) pointer-set.h lists.o: lists.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) $(DIAGNOSTIC_CORE_H) \ $(RTL_H) $(GGC_H) gt-lists.h bb-reorder.o : bb-reorder.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \ Index: gcc/cgraphclones.c =================================================================== --- gcc/cgraphclones.c (revision 201022) +++ gcc/cgraphclones.c (working copy) @@ -94,7 +94,6 @@ along with GCC; see the file COPYING3. If not see #include "ipa-utils.h" #include "lto-streamer.h" #include "except.h" -#include "auto-profile.h" /* Create clone of E in the node N represented by CALL_EXPR the callgraph. */ struct cgraph_edge * @@ -278,9 +277,6 @@ clone_function_name (tree decl, const char *suffix prefix[len] = '_'; #endif ASM_FORMAT_PRIVATE_NAME (tmp_name, prefix, clone_fn_id_num++); - if (flag_auto_profile) - afdo_add_bfd_name_mapping (xstrdup (tmp_name), - xstrdup (lang_hooks.dwarf_name (decl, 0))); return get_identifier (tmp_name); } Index: gcc/value-prof.c =================================================================== --- gcc/value-prof.c (revision 201022) +++ gcc/value-prof.c (working copy) @@ -1455,8 +1455,7 @@ gimple_ic (gimple icall_stmt, struct cgraph_node * /* Build an EH edge for the direct call if necessary. */ lp_nr = lookup_stmt_eh_lp (icall_stmt); - if (lp_nr != 0 - && stmt_could_throw_p (dcall_stmt)) + if (lp_nr > 0 && stmt_could_throw_p (dcall_stmt)) { edge e_eh, e; edge_iterator ei; Index: gcc/cgraphunit.c =================================================================== --- gcc/cgraphunit.c (revision 201022) +++ gcc/cgraphunit.c (working copy) @@ -195,7 +195,6 @@ along with GCC; see the file COPYING3. If not see #include "l-ipo.h" #include "except.h" #include "regset.h" /* FIXME: For reg_obstack. */ -#include "auto-profile.h" /* Queue of cgraph nodes scheduled to be added into cgraph. This is a secondary queue used during optimization to accommodate passes that @@ -2243,13 +2242,6 @@ finalize_compilation_unit (void) { timevar_push (TV_CGRAPH); - /* Before compilation, auto profile will process the profile to build the - hash tables for later optimizations. We delay this function call here - because all the parsing should be done so that we will have the bfd - name mapping ready. */ - if (flag_auto_profile) - process_auto_profile (); - /* If we're here there's no current function anymore. Some frontends are lazy in clearing these. */ current_function_decl = NULL; Index: gcc/ipa-inline-transform.c =================================================================== --- gcc/ipa-inline-transform.c (revision 201022) +++ gcc/ipa-inline-transform.c (working copy) @@ -312,8 +312,8 @@ dump_inline_decision (struct cgraph_edge *edge) dump_printf_loc (is_in_ipa_inline ? MSG_OPTIMIZED_LOCATIONS : MSG_NOTE, locus, "%s inlined into %s%s%s\n", - cgraph_node_opt_info (edge->callee, true), - cgraph_node_opt_info (final_caller, true), + cgraph_node_opt_info (edge->callee, false), + cgraph_node_opt_info (final_caller, false), call_count_text, inline_chain_text); } Index: gcc/auto-profile.c =================================================================== --- gcc/auto-profile.c (revision 201022) +++ gcc/auto-profile.c (working copy) @@ -22,6 +22,8 @@ along with GCC; see the file COPYING3. If not see file. */ #include +#include +#include #include "config.h" #include "system.h" @@ -53,25 +55,22 @@ along with GCC; see the file COPYING3. If not see Phase 1: Read profile from the profile data file. The following info is read from the profile datafile: - * Function names and file names. - * Source level profile, which is a mapping from inline stack to - its sample counts. - * Module profile: Module to aux-modules mapping - Phase 1 just reads in data without processing it. It is invoked - before tree parsing because LIPO needs module profile before tree - parsing. (read_aux_modules) + * StringMap: a map between function name and its index. + * SymbolMap: a map from symbol name to Symbol. This is represented as + a forest of Symbols. + * ModuleMap: a map from module name to its compilation/aux-module info. + * WorkingSet: a histogram of how many instructions are covered for a + given percentage of total cycles. - Phase 2: Process profile to build internal data structure (hashmap). - This is done after tree parsing, because the processing requires the map - from function name to its debug name (bfd_name). The following hashmaps - is used to store profile. - * function_htab: map from function_name to its entry_bb count - * stack_htab: map from inline stack to its sample count - * bfd_name_htab: map from function name to its debug name (bfd_name) - * module_htab: map from module name to its aux-module names + Phase 2: Early inline. + Early inline uses SymbolMap to find if a callsite is: + * inlined in the profiled binary. + * callee body is hot in the profiling run. + If both condition satisfies, early inline will inline the callsite + regardless of the code growth. Phase 3: Annotate control flow graph. - AutoFDO invokes a separate pass over the control flow graph to: + AutoFDO uses a separate pass to: * Annotate basic block count * Estimate branch probability @@ -79,563 +78,195 @@ along with GCC; see the file COPYING3. If not see AutoFDO tries to reuse all FDO infrastructure as much as possible to make use of the profile. E.g. it uses existing mechanism to calculate the basic block/edge frequency, as well as the cgraph node/edge count. - - However, AutoFDO still differs from FDO in the following aspects: - - * Profile is not accurate, because AutoFDO uses sampling to collect - profile, and uses debug info to represent the profile. As a result, - some hot basic blocks may have zero sample count. Because of this, - some optimization needs to be adjusted (e.g. loop peeling/unrolling). - * Each cloned context has its own profile, but these contexts may - not even exist when doing annotation. This provides more context- - sensitive profiles, but at the same time, adds complexity to the - implementation. Because of this, additional profile annotation is - needed for each function after the inline pass, and count scaling - is tricky in the second annotation. */ #define DEFAULT_AUTO_PROFILE_FILE "fbdata.afdo" -#define SP_HTAB_INIT_SIZE 2000 -/* GCOV data structures to represent profile stored in the .afdo file. */ - -struct gcov_callsite_pos +struct SourceLocation { - const char *file; - const char *func; - gcov_unsigned_t line; - gcov_unsigned_t discr; + tree func_decl; + unsigned lineno; }; -struct gcov_stack +typedef std::vector StringVector; +typedef std::vector InlineStack; +typedef std::map TargetMap; + +struct ProfileInfo { - const char *func_name; - const char *callee_name; - struct gcov_callsite_pos *stack; - gcov_unsigned_t size; - struct gcov_hist *hist; - gcov_unsigned_t hist_size; - gcov_type num_inst; gcov_type count; - gcov_type max_count; + TargetMap target_map; }; -struct gcov_function +struct StringCompare { - const char *name; - const char *file; - gcov_type total_count; - gcov_type entry_count; - gcov_type max_count; - /* Number of call stacks in the function. */ - gcov_unsigned_t stack_num; - /* All the call stacks in the function. */ - struct gcov_stack *stacks; + bool operator() (const char* a, const char* b) const + { return strcmp (a, b) < 0; } }; + +class StringMap { + public: + static StringMap *Create(); + int GetIndex (const char *name) const; + int GetIndexByDecl (tree decl) const; + const char *GetName (int index) const; -struct afdo_bfd_name -{ - const char *assembler_name; - /* bfd_name is the name that debugger used for function name matching. - Different assembler names could map to the same bfd_name. */ - const char *bfd_name; -}; + private: + StringMap () {} + bool Read (); -struct afdo_module -{ - char *name; - int ident; - unsigned exported; - unsigned lang; - unsigned ggc_memory; - unsigned num_aux_modules; - unsigned num_quote_paths; - unsigned num_bracket_paths; - unsigned num_cpp_defines; - unsigned num_cpp_includes; - unsigned num_cl_args; - char **strings; + typedef std::map StringIndexMap; + StringVector vector_; + StringIndexMap map_; }; -struct gcov_hist -{ - enum hist_type type; - union - { - const char *func_name; - unsigned long long value; - } value; - gcov_type count; -}; +class Symbol { + public: + typedef std::vector SymbolStack; -/* Store the file name strings read from the profile data file. */ -static const char **file_names; + /* Read the profile and create a symbol with head count as HEAD_COUNT. + Recursively read callsites to create nested symbols too. STACK is used + to track the recursive creation process. */ + static const Symbol *ReadSymbol (SymbolStack *stack, gcov_type head_count); -/* gcov_ctr_summary structure to store the profile_info. */ -static struct gcov_ctr_summary *afdo_profile_info; + /* Recursively deallocate all callsites (nested symbols). */ + ~Symbol (); -/* Hash table to hold function information. */ -static htab_t function_htab; + /* Accessors. */ + unsigned name () const { return name_; } + gcov_type total_count () const { return total_count_; } + gcov_type head_count () const { return head_count_; } -/* Hash table to hold stack information. */ -static htab_t stack_htab; + /* Recursively traverse STACK starting from LEVEL to find the corresponding + symbol. */ + const Symbol *GetSymbol (const InlineStack &stack, unsigned level) const; -/* Hash table to hold assembler name to bfd name mapping. */ -static htab_t bfd_name_htab; + /* Return the profile info for LOC. */ + bool GetProfileInfo (location_t loc, ProfileInfo *info) const; -/* Hash table to hold module informaition. */ -static htab_t module_htab; + private: + Symbol (unsigned name, gcov_type head_count) + : name_(name), total_count_(0), head_count_(head_count) {} + const Symbol *GetSymbolByDecl (unsigned lineno, tree decl) const; -/* Store the module hash table contents. */ -static struct afdo_module *modules; + typedef std::map CallsiteMap; + typedef std::map PositionCountMap; -/* File static variables, which is used to pass information between - init_auto_profile and process_auto_profile. */ -static gcov_unsigned_t function_num; -static gcov_unsigned_t total_module_num; -static struct gcov_function *gcov_functions; + /* Symbol name index in the string map. */ + unsigned name_; + /* The total sampled count. */ + gcov_type total_count_; + /* The total sampled count in the head bb. */ + gcov_type head_count_; + /* Map from callsite location to callee symbol. */ + CallsiteMap callsites; + /* Map from source location to count and instruction number. */ + PositionCountMap pos_counts; +}; -/* Check if PATH_NAME is absolute path, if yes, strip the directory part - of the PATH_NAME, return the file name. */ - -static const char * -afdo_get_filename (const char *path_name) -{ - const char* last; - return path_name; - if (path_name == NULL) - return NULL; - last = strrchr (path_name, '/'); - return ((last == 0) ? path_name : last + 1); -} - -/* Given an assembler function NAME, return its original name. strip the - suffix at the end of the function name, added by optimizations such as - constant propagation etc. */ - -static gcov_unsigned_t -afdo_get_original_name_size (const char *name) -{ - const char *ret; - if (!name) - return 0; - ret = strchr (name, '.'); - if (!ret) - return strlen (name); - else - return ret - name; -} - -/* Given an asssembler function NAME, return its corresponding bfd name. - If the mapping cannot be found, it means that the assembler function - name is not used/emitted in the current module(s). */ - -static const char * -afdo_get_bfd_name (const char *name) -{ - struct afdo_bfd_name bfd, *bfd_entry; - gcov_unsigned_t size = afdo_get_original_name_size (name); - /* If the function name is cloned, we want to find its original name. */ - char *buf = (char *) alloca (size + 1); - strncpy (buf, name, size); - buf[size] = 0; - bfd.assembler_name = buf; - bfd_entry = (struct afdo_bfd_name *) htab_find (bfd_name_htab, &bfd); - if (!bfd_entry) - return name; - return bfd_entry->bfd_name; -} - -/* Traverse the cgraph, add each function's name to to bfd_name mapping. */ - -static void -afdo_read_bfd_names (void) -{ - struct cgraph_node *node; - - FOR_EACH_FUNCTION (node) +class SymbolMap { + public: + static SymbolMap *Create () { - const char *bfd_name; - if (lang_hooks.dwarf_name (node->symbol.decl, 0) == NULL) - continue; - bfd_name = xstrdup (lang_hooks.dwarf_name (node->symbol.decl, 0)); - afdo_add_bfd_name_mapping (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME - (node->symbol.decl)), bfd_name); + SymbolMap *map = new SymbolMap (); + if (map->Read ()) + return map; + delete map; + return NULL; } -} + ~SymbolMap (); + const Symbol *GetSymbolByDecl (tree decl) const; + bool GetProfileInfo (gimple stmt, ProfileInfo *info) const; + gcov_type GetCallsiteTotalCount (struct cgraph_edge *edge) const; -/* Hash function for struct afdo_stack. */ + private: + typedef std::map NameSymbolMap; -static hashval_t -afdo_stack_hash (const void *stack) -{ - gcov_unsigned_t i; - /* An arbitrary initial value borrowed from hashtab.c. */ - hashval_t h = 0x9e3779b9; - const struct gcov_stack *s = (const struct gcov_stack *) stack; - if (s->callee_name) - h = iterative_hash (afdo_get_bfd_name (s->callee_name), - strlen (afdo_get_bfd_name (s->callee_name)), h); - if (s->func_name) - h = iterative_hash (s->func_name, - afdo_get_original_name_size (s->func_name), h); - for (i = 0; i < s->size; i++) { - const struct gcov_callsite_pos *p = s->stack + i; - const char *file = afdo_get_filename (p->file); - const char *func = afdo_get_bfd_name (p->func); - h = iterative_hash (file, strlen (file), h); - if (func) - h = iterative_hash (func, strlen (func), h); - h = iterative_hash (&p->line, sizeof (p->line), h); - if (i == 0) - h = iterative_hash (&p->discr, sizeof (p->discr), h); - } - return h; -} + SymbolMap () {} + bool Read (); + const Symbol *GetSymbolByInlineStack (const InlineStack &stack) const; -/* Check if two afdo_stack P and Q are identical. */ + NameSymbolMap map_; +}; -static int -afdo_stack_eq (const void *p, const void *q) -{ - const struct gcov_stack *s1 = (const struct gcov_stack *) p; - const struct gcov_stack *s2 = (const struct gcov_stack *) q; - - gcov_unsigned_t i; - if (s1->func_name == NULL || s2->func_name == NULL) - return 0; - - if (s1->callee_name == NULL) +class ModuleMap { + public: + static ModuleMap *Create () { - if (s2->callee_name != NULL) - return 0; + ModuleMap *map = new ModuleMap (); + if (map->Read ()) + return map; + delete map; + return NULL; } - else if (s2->callee_name == NULL) - return 0; - else if (strcmp (afdo_get_bfd_name (s1->callee_name), - afdo_get_bfd_name (s2->callee_name))) - return 0; - i = afdo_get_original_name_size (s1->func_name); - if (i != afdo_get_original_name_size (s2->func_name)) - return 0; - - if (strncmp (s1->func_name, s2->func_name, i)) - return 0; - - if (s1->size != s2->size) - return 0; - for (i = 0; i < s1->size; i++) + gcov_module_info *GetModule(const char *name) const { - const struct gcov_callsite_pos *p1 = s1->stack + i; - const struct gcov_callsite_pos *p2 = s2->stack + i; - const char *func1 = afdo_get_bfd_name (p1->func); - const char *func2 = afdo_get_bfd_name (p2->func); - - if (func1 != NULL && func2 != NULL) - { - if (strcmp (func1, func2)) - return 0; - } - else if (func1 != func2) - return 0; - - if (strcmp (afdo_get_filename (p1->file), afdo_get_filename (p2->file)) - || p1->line != p2->line || (i== 0 && p1->discr != p2->discr)) - return 0; + NameModuleMap::const_iterator iter = map_.find (name); + return iter == map_.end() ? NULL : iter->second.second; } - return 1; -} -/* Hash function for struct afdo_function. */ - -static hashval_t -afdo_function_hash (const void *func) -{ - /* An arbitrary initial value borrowed from hashtab.c. */ - hashval_t h = 0x9e3779b9; - const struct gcov_function *f = (const struct gcov_function *) func; - - if (f->name) - h = iterative_hash (f->name, afdo_get_original_name_size (f->name), h); - return h; -} - -/* Check if two afdo_function P and Q are identical. */ - -static int -afdo_function_eq (const void *p, const void *q) -{ - const struct gcov_function *f1 = (const struct gcov_function *) p; - const struct gcov_function *f2 = (const struct gcov_function *) q; - gcov_unsigned_t i; - - if (f1->name == NULL || f2->name == NULL) - return 0; - - i = afdo_get_original_name_size (f1->name); - if (i != afdo_get_original_name_size (f2->name)) - return 0; - - return !strncmp (f1->name, f2->name, i); -} - -/* Hash function for struct afdo_bfd_name. */ - -static hashval_t -afdo_bfd_name_hash (const void *func) -{ - hashval_t h = 0x9e3779b9; - const struct afdo_bfd_name *f = (const struct afdo_bfd_name *) func; - - if (f->assembler_name) - h = iterative_hash (f->assembler_name, strlen (f->assembler_name), h); - return h; -} - -/* Check if two struct afdo_bfd_name P and Q are identical. */ - -static int -afdo_bfd_name_eq (const void *p, const void *q) -{ - const struct afdo_bfd_name *b1 = (const struct afdo_bfd_name *) p; - const struct afdo_bfd_name *b2 = (const struct afdo_bfd_name *) q; - - if (b1->assembler_name == NULL || b2->assembler_name == NULL) - return 0; - - return !strcmp (b1->assembler_name, b2->assembler_name); -} - -/* Free the hash table entry P. */ - -static void -afdo_bfd_name_del (void *p) -{ - free (p); -} - -/* Hash Function for struct afdo_module. */ - -static hashval_t -afdo_module_hash (const void *module) -{ - hashval_t h = 0x9e3779b9; - const struct afdo_module *m = (const struct afdo_module *)module; - - if (m->name) - h = iterative_hash (m->name, strlen (m->name), h); - - return h; -} - -/* Check if two struct afdo_module P and Q are identical. */ - -static int -afdo_module_eq (const void *p, const void *q) -{ - const struct afdo_module *m1 = (const struct afdo_module *)p; - const struct afdo_module *m2 = (const struct afdo_module *)q; - - if (m1->name == NULL || m2->name == NULL) - return 0; - - return !strcmp (m1->name, m2->name); -} - -/* Return the total number of emitted string for MODULE. */ - -static unsigned long long -afdo_module_num_strings (const struct afdo_module *module) -{ - return module->num_quote_paths + - module->num_bracket_paths + - module->num_cpp_defines + - module->num_cpp_includes + - module->num_cl_args; -} - -/* Add a module (specified in MODULE) into gcov_module_info format in - MODULE_INFO, which is used by LIPO to import auxiliary modules. - Set the is_primary flag if IS_PRIMARY is set. */ - -static void -afdo_add_module (struct gcov_module_info **module_info, - const struct afdo_module *module, - gcov_unsigned_t is_primary) -{ - unsigned i; - size_t info_sz; - - info_sz = sizeof (struct gcov_module_info) + - sizeof (void *) * afdo_module_num_strings (module); - *module_info = XCNEWVAR (struct gcov_module_info, info_sz); - (*module_info)->ident = module->ident; - (*module_info)->is_primary = is_primary; - (*module_info)->flags = is_primary ? module->exported : 1; - (*module_info)->lang = module->lang; - (*module_info)->ggc_memory = module->ggc_memory; - (*module_info)->source_filename = module->name; - (*module_info)->num_quote_paths = module->num_quote_paths; - (*module_info)->num_bracket_paths = module->num_bracket_paths; - (*module_info)->num_cpp_defines = module->num_cpp_defines; - (*module_info)->num_cpp_includes = module->num_cpp_includes; - (*module_info)->num_cl_args = module->num_cl_args; - for (i = 0; i < afdo_module_num_strings (module); i++) - (*module_info)->string_array[i] = - module->strings[module->num_aux_modules + i]; -} - -/* Read in the auxiliary modules for the current primary module. */ - -static void -read_aux_modules (void) -{ - unsigned i, curr_module = 1; - struct afdo_module module, *entry; - - module.name = xstrdup (in_fnames[0]); - entry = (struct afdo_module *) htab_find (module_htab, &module); - if (!entry) - return; - module_infos = XCNEWVEC (struct gcov_module_info *, - entry->num_aux_modules + 1); - afdo_add_module (module_infos, entry, true); - primary_module_id = entry->ident; - for (i = 0; i < entry->num_aux_modules; i++) + const StringVector *GetAuxModules(const char *name) const { - struct afdo_module *aux_entry; - module.name = entry->strings[i]; - if (!strcmp (module.name, in_fnames[0])) - continue; - aux_entry = (struct afdo_module *) htab_find (module_htab, &module); - if (!aux_entry) - { - inform (0, "aux module %s cannot be found.", module.name); - continue; - } - if ((aux_entry->lang & GCOV_MODULE_LANG_MASK) != - (entry->lang & GCOV_MODULE_LANG_MASK)) - { - inform (0, "Not importing %s: source language" - " different from primary module's source language", - aux_entry->name); - continue; - } - if ((aux_entry->lang & GCOV_MODULE_ASM_STMTS) - && flag_ripa_disallow_asm_modules) - { - if (flag_opt_info) - inform (0, "Not importing %s: contains " - "assembler statements", aux_entry->name); - continue; - } - afdo_add_module (&module_infos[curr_module], aux_entry, false); - if (incompatible_cl_args (module_infos[0], module_infos[curr_module])) - { - if (flag_opt_info) - inform (0, "Not importing %s: command-line" - " arguments not compatible with primary module", - aux_entry->name); - free (module_infos[curr_module]); - continue; - } - else - { - curr_module ++; - add_input_filename (module.name); - } + NameModuleMap::const_iterator iter = map_.find (name); + return iter == map_.end() ? NULL : &iter->second.first; } -} -/* From AutoFDO profiles, find values inside STMT for that we want to measure - histograms for indirect-call optimization. */ + private: + ModuleMap () {} + bool Read (); -static void -afdo_indirect_call (gimple stmt, struct gcov_hist *values, int hist_size) -{ - tree callee; - int i, total = 0; - int actual_count = 0; - histogram_value hist; + typedef std::pair AuxInfo; + typedef std::map NameModuleMap; + NameModuleMap map_; +}; - if (gimple_code (stmt) != GIMPLE_CALL - || gimple_call_fndecl (stmt) != NULL_TREE) - return; - callee = gimple_call_fn (stmt); +/* Store the strings read from the profile data file. */ +static StringMap *string_map; +static SymbolMap *symbol_map; +static ModuleMap *module_map; - for (i = 0; i < hist_size; i++) - if (values[i].type == HIST_TYPE_INDIR_CALL_TOPN) - break; +/* gcov_ctr_summary structure to store the profile_info. */ +static struct gcov_ctr_summary *afdo_profile_info; - if (i == hist_size) - return; - hist = gimple_alloc_histogram_value (cfun, HIST_TYPE_INDIR_CALL_TOPN, - stmt, callee); - hist->n_counters = (GCOV_ICALL_TOPN_VAL << 2) + 1; - hist->hvalue.counters = XNEWVEC (gcov_type, hist->n_counters); - gimple_add_histogram_value (cfun, stmt, hist); +/* Helper functions. */ - for (i = 0; i < hist_size; i++) - if (values[i].type == HIST_TYPE_INDIR_CALL_TOPN) - { - total += values[i].count; - /* Values are pre-sorted by the profile generator. */ - if (actual_count < 2) - { - hist->hvalue.counters[actual_count * 2 + 1] = - (unsigned long long) values[i].value.func_name; - hist->hvalue.counters[actual_count * 2 + 2] = values[i].count; - actual_count ++; - } - } +/* Return a callsite represented as a 64bit integer, in which the + higher 32 bits represent OFFSET (combined location), the lower + 32 bits represent CALLEE_NAME index in string_map. */ - hist->hvalue.counters[0] = total; - - if (actual_count == 1) - { - hist->hvalue.counters[3] = 0; - hist->hvalue.counters[4] = 0; - } +static gcov_type get_callsite (unsigned offset, unsigned callee_name) +{ + return ((gcov_type) offset) << 32 | callee_name; } -/* From AutoFDO profiles, find values inside STMT for that we want to measure - histograms and adds them to list VALUES. */ +/* Return the original name of NAME: strip the suffix that starts + with '.' */ -static void -afdo_vpt (gimple stmt, struct gcov_hist *v, int hist_size) +static const char *get_original_name (const char *name) { - afdo_indirect_call (stmt, v, hist_size); + char *ret = xstrdup (name); + char *find = strchr (ret, '.'); + if (find != NULL) + *find = 0; + return ret; } -/* Return the size of the inline stack of the STMT. */ +/* Return the combined location, which is a 32bit integer in which + higher 16 bits stores the line offset of LOC to the start lineno + of DECL, The lower 16 bits stores the discrimnator of LOC if + USE_DISCR is true, otherwise 0. */ -static int -get_inline_stack_size_by_stmt (gimple stmt) +static unsigned +get_combined_location (location_t loc, tree decl, bool use_discr) { - tree block; - int size = 1; - - if (!stmt) - return 0; - if (LOCATION_LOCUS (gimple_location (stmt)) == UNKNOWN_LOCATION) - return 0; - block = gimple_block (stmt); - if (!block || TREE_CODE (block) != BLOCK || !gimple_location (stmt)) - return 0; - - for ( block = BLOCK_SUPERCONTEXT (block); - block && (TREE_CODE (block) == BLOCK); - block = BLOCK_SUPERCONTEXT (block)) { - /* Traverse the nesting blocks. If the block contains the source - location info, save the source location info to the inline stack. */ - if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block)) == UNKNOWN_LOCATION) - continue; - size ++; - } - return size; + if (use_discr) + return ((LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl)) << 16) + | get_discriminator_from_locus (loc); + else + return (LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl)) << 16; } /* Return the function decl of a given lexical BLOCK. */ @@ -656,296 +287,333 @@ get_function_decl_from_block (tree block) return decl; } -/* Store the inline stack of STMT to POS_STACK, return the size of the - stack. Set the discriminator of the inline stack if DISCR is TRUE. */ - -static int -get_inline_stack_by_stmt (gimple stmt, tree decl, - struct gcov_callsite_pos *pos_stack, bool discr) +static void +get_inline_stack (gimple stmt, bool use_discr, InlineStack *stack) { - tree block; - int idx = 0; - source_location loc; + location_t locus = gimple_location (stmt); + if (LOCATION_LOCUS (locus) == UNKNOWN_LOCATION) + return; - if (!stmt) - return 0; - block = gimple_block (stmt); - if (!block || TREE_CODE (block) != BLOCK || !gimple_location (stmt)) - return 0; + tree block = gimple_block (stmt); + if (!block || TREE_CODE (block) != BLOCK) + return; - loc = gimple_location (stmt); - if (LOCATION_LOCUS (loc) == UNKNOWN_LOCATION) - return 0; - pos_stack[idx].file = expand_location (loc).file; - pos_stack[idx].line = expand_location (loc).line; - if (discr) - pos_stack[idx].discr = get_discriminator_from_locus (loc); - else - pos_stack[idx].discr = 0; - idx++; + int level = 0; for (block = BLOCK_SUPERCONTEXT (block); block && (TREE_CODE (block) == BLOCK); block = BLOCK_SUPERCONTEXT (block)) { - tree decl; - loc = BLOCK_SOURCE_LOCATION (block); - - if (LOCATION_LOCUS (loc) == UNKNOWN_LOCATION) + location_t tmp_locus = BLOCK_SOURCE_LOCATION (block); + if (LOCATION_LOCUS (tmp_locus) == UNKNOWN_LOCATION) continue; - decl = get_function_decl_from_block (block); - pos_stack[idx].file = expand_location (loc).file; - pos_stack[idx].line = expand_location (loc).line; - pos_stack[idx - 1].func = - decl ? IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)) : NULL; - pos_stack[idx - 1].line -= decl ? DECL_SOURCE_LINE (decl) : 0; - pos_stack[idx++].discr = 0; + + tree decl = get_function_decl_from_block (block); + SourceLocation src_loc; + src_loc.func_decl = decl; + src_loc.lineno = get_combined_location (locus, decl, + level == 0 && use_discr); + stack->push_back (src_loc); + locus = tmp_locus; + level++; } - if (decl) - { - pos_stack[idx - 1].func = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)); - pos_stack[idx - 1].line -= DECL_SOURCE_LINE (decl); - } - return idx; + SourceLocation src_loc; + src_loc.func_decl = current_function_decl; + src_loc.lineno = get_combined_location (locus, current_function_decl, + level == 0 && use_discr); + stack->push_back (src_loc); } -/* Read sample count info of the function with DECL, and save them - to ENTRY_COUNT and TOTAL_COUNT respectively. */ -static void -afdo_get_function_count (tree decl, - gcov_type *entry_count) +/* Member functions for StringMap. */ + +StringMap *StringMap::Create() { - struct gcov_function func; - const struct gcov_function *func_entry; + StringMap *map = new StringMap(); + if (map->Read()) + return map; + delete map; + return NULL; +} - *entry_count = 0; - func.name = - IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)); - func.file = DECL_SOURCE_FILE (decl); - func_entry = (const struct gcov_function *) - htab_find (function_htab, &func); - if (func_entry) - { - /* We need to use the sum because in the profile collection binary, - there are many cloned functions such as isra functions. We want - to combine their profiles. */ - (*entry_count) = func_entry->entry_count; - return; - } - func.name = afdo_get_bfd_name (func.name); - func_entry = (const struct gcov_function *) - htab_find (function_htab, &func); - if (func_entry) - (*entry_count) = func_entry->entry_count; +int StringMap::GetIndex (const char *name) const +{ + if (name == NULL) + return -1; + StringIndexMap::const_iterator iter = map_.find (name); + if (iter == map_.end()) + return -1; + else + return iter->second; } -/* Set the node count of the current function, and update the entry_bb - count. */ +int StringMap::GetIndexByDecl (tree decl) const +{ + const char *name = get_original_name ( + IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl))); + int ret = GetIndex (name); + if (ret != -1) + return ret; + ret = GetIndex (lang_hooks.dwarf_name (decl, 0)); + if (ret != -1) + return ret; + if (DECL_ABSTRACT_ORIGIN (decl)) + return GetIndexByDecl (DECL_ABSTRACT_ORIGIN (decl)); + else + return -1; +} -void -afdo_set_current_function_count (void) +const char *StringMap::GetName (int index) const { - gcov_type entry_count; - struct cgraph_node *node = cgraph_get_create_node (current_function_decl); + gcc_assert (index > 0 && index < (int) vector_.size()); + return vector_[index]; +} - afdo_get_function_count (current_function_decl, &entry_count); - node->count = entry_count; - ENTRY_BLOCK_PTR->count = node->count; +bool StringMap::Read () +{ + if (gcov_read_unsigned () != GCOV_TAG_AFDO_FILE_NAMES) + return false; + /* Skip the length of the section. */ + gcov_read_unsigned (); + /* Read in the file name table. */ + unsigned string_num = gcov_read_unsigned (); + for (unsigned i = 0; i < string_num; i++) + { + vector_.push_back (get_original_name (gcov_read_string ())); + map_[vector_.back()] = i; + } + return true; } -/* Add the AS_NAME->BFD_NAME to the assembler_name to bfd_name mapping. */ -void -afdo_add_bfd_name_mapping (const char *as_name, const char *bfd_name) +/* Member functions for Symbol. */ + +Symbol::~Symbol () { - struct afdo_bfd_name **slot; - struct afdo_bfd_name *entry = (struct afdo_bfd_name *) - xmalloc (sizeof (struct afdo_bfd_name)); + for (CallsiteMap::iterator iter = callsites.begin(); + iter != callsites.end(); ++iter) + delete iter->second; +} - entry->assembler_name = as_name; - entry->bfd_name = bfd_name; - slot = (struct afdo_bfd_name **) - htab_find_slot (bfd_name_htab, entry, INSERT); - if (!*slot) - *slot = entry; +const Symbol *Symbol::GetSymbolByDecl (unsigned lineno, tree decl) const +{ + int func_name_idx = string_map->GetIndexByDecl (decl); + if (func_name_idx != -1) + { + CallsiteMap::const_iterator ret = callsites.find ( + get_callsite (lineno, func_name_idx)); + if (ret != callsites.end ()) + return ret->second; + } + func_name_idx = string_map->GetIndex (lang_hooks.dwarf_name (decl, 0)); + if (func_name_idx != -1) + { + CallsiteMap::const_iterator ret = callsites.find ( + get_callsite (lineno, func_name_idx)); + if (ret != callsites.end ()) + return ret->second; + } + if (DECL_ABSTRACT_ORIGIN (decl)) + return GetSymbolByDecl (lineno, DECL_ABSTRACT_ORIGIN (decl)); else - free (entry); + return NULL; } -/* For a given POS_STACK with SIZE, get the COUNT, MAX_COUNT, NUM_INST, - HIST_SIZE and HIST for the inline stack. If CALLEE_NAME is non-null, - the COUNT/MAX_COUNT represents the total/max count in the inline stack. - Otherwise, the COUNT represents the count of an ordinary statement, - HIST stores the value histogram vectors with size of HIST_SIZE. - Return FALSE if profile is not found for the given POS_STACK. */ +const Symbol *Symbol::GetSymbol (const InlineStack &stack, unsigned level) const +{ + if (level == 0) + return this; + const Symbol *symbol = + GetSymbolByDecl (stack[level].lineno, stack[level - 1].func_decl); + if (symbol) + return symbol->GetSymbol (stack, level - 1); + else + return NULL; +} -static bool -get_stack_count (struct gcov_callsite_pos *pos_stack, - const char *callee_name, int size, - gcov_type *count, gcov_type *max_count, gcov_type *num_inst, - gcov_unsigned_t *hist_size, struct gcov_hist **hist) +bool Symbol::GetProfileInfo (location_t loc, ProfileInfo *info) const { - struct gcov_stack stack, *entry; - stack.func_name = pos_stack[size - 1].func; - stack.callee_name = callee_name; - stack.stack = pos_stack; - stack.size = size; - entry = (struct gcov_stack *) htab_find (stack_htab, &stack); - if (entry) + PositionCountMap::const_iterator iter = pos_counts.find (loc); + if (iter == pos_counts.end ()) + return false; + *info = iter->second; + return true; +} + +const Symbol *Symbol::ReadSymbol (SymbolStack *stack, gcov_type head_count) +{ + unsigned name = gcov_read_unsigned (); + unsigned num_pos_counts = gcov_read_unsigned (); + unsigned num_callsites = gcov_read_unsigned (); + Symbol *symbol = new Symbol (name, head_count); + stack->push_back(symbol); + + for (unsigned i = 0; i < num_pos_counts; i++) { - *count = entry->count; - *num_inst = entry->num_inst; - if (max_count) - *max_count = entry->max_count; - if (hist_size) + unsigned offset = gcov_read_unsigned (); + unsigned num_targets = gcov_read_unsigned (); + gcov_type count = gcov_read_counter (); + symbol->pos_counts[offset].count = count; + for (unsigned j = 0; j < stack->size(); j++) + (*stack)[j]->total_count_ += count; + for (unsigned j = 0; j < num_targets; j++) { - *hist_size = entry->hist_size; - *hist = entry->hist; + /* Only indirect call target histogram is supported now. */ + gcov_read_unsigned (); + gcov_type target_idx = gcov_read_counter (); + symbol->pos_counts[offset].target_map[target_idx] = + gcov_read_counter (); } - return true; } - *count = 0; - *num_inst = 0; - if (max_count) - *max_count = 0; - if (hist_size) - { - *hist_size = 0; - *hist = 0; - } - return false; + for (unsigned i = 0; i < num_callsites; i++) { + unsigned offset = gcov_read_unsigned (); + const Symbol *callee_symbol = ReadSymbol (stack, 0); + symbol->callsites[get_callsite (offset, callee_symbol->name ())] = + callee_symbol; + } + stack->pop_back(); + return symbol; } -/* For a given STMT, get the COUNT and NUM_INST from its profile. - Return FALSE if profile is not found for STMT. */ -static bool -get_stmt_count (gimple stmt, gcov_type *count, gcov_type *num_inst, - gcov_unsigned_t *hist_size, struct gcov_hist **hist) +/* Member functions for SymbolMap. */ + +SymbolMap::~SymbolMap () { - struct gcov_callsite_pos *pos_stack; - int size; + for (NameSymbolMap::const_iterator iter = map_.begin (); + iter != map_.end (); ++iter) + delete iter->second; +} - if (!stmt) - return false; - size = get_inline_stack_size_by_stmt (stmt); - if (size == 0) - return false; +const Symbol *SymbolMap::GetSymbolByDecl (tree decl) const +{ + int index = string_map->GetIndexByDecl (decl); + if (index == -1) + return NULL; + NameSymbolMap::const_iterator ret = map_.find (index); + return ret == map_.end() ? NULL : ret->second; +} + +bool SymbolMap::GetProfileInfo (gimple stmt, ProfileInfo *info) const +{ if (LOCATION_LOCUS (gimple_location (stmt)) == cfun->function_end_locus) return false; - pos_stack = (struct gcov_callsite_pos *) - alloca (sizeof (struct gcov_callsite_pos) * size); - - get_inline_stack_by_stmt (stmt, current_function_decl, pos_stack, true); - - return get_stack_count (pos_stack, NULL, size, count, NULL, num_inst, - hist_size, hist); + InlineStack stack; + get_inline_stack (stmt, true, &stack); + if (stack.size () == 0) + return false; + const Symbol *symbol = GetSymbolByInlineStack (stack); + if (symbol == NULL) + return false; + return symbol->GetProfileInfo (stack[0].lineno, info); } -/* For a given EDGE, if IS_TOTAL is true, save EDGE->callee's total count - to COUNT, otherwise save EDGE's count to COUNT. */ - -static bool -get_callsite_count (struct cgraph_edge *edge, gcov_type *count, - gcov_type *max_count) +gcov_type SymbolMap::GetCallsiteTotalCount (struct cgraph_edge *edge) const { - struct gcov_callsite_pos *pos_stack; - gcov_type num_inst; - const char *callee_name = - IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (edge->callee->symbol.decl)); - int size = get_inline_stack_size_by_stmt (edge->call_stmt); + InlineStack stack; + SourceLocation src_loc; + src_loc.func_decl = edge->callee->symbol.decl; + stack.push_back (src_loc); + get_inline_stack (edge->call_stmt, false, &stack); - if (size == 0) + const Symbol *symbol = GetSymbolByInlineStack (stack); + if (symbol == NULL) return 0; - pos_stack = (struct gcov_callsite_pos *) - alloca (sizeof (struct gcov_callsite_pos) * size); - - get_inline_stack_by_stmt (edge->call_stmt, edge->caller->symbol.decl, - pos_stack, false); - - return get_stack_count (pos_stack, callee_name, - size, count, max_count, &num_inst, NULL, NULL); + else + return symbol->total_count (); } -/* For a given BB, return its execution count, and annotate value profile - on statements. */ - -static gcov_type -afdo_get_bb_count (basic_block bb) +bool SymbolMap::Read () { - gimple_stmt_iterator gsi; - gcov_type max_count = 0; - bool has_annotated = false; - - for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) + if (gcov_read_unsigned () != GCOV_TAG_AFDO_FUNCTION) { - gcov_type count, num_inst; - gcov_unsigned_t hist_size; - struct gcov_hist *hist; - gimple stmt = gsi_stmt (gsi); - if (get_stmt_count (stmt, &count, &num_inst, &hist_size, &hist)) - { - if (count > max_count) - max_count = count; - has_annotated = true; - if (hist_size > 0) - afdo_vpt (stmt, hist, hist_size); - } + inform (0, "Not expected TAG."); + return false; } - if (has_annotated) + + /* Skip the length of the section. */ + gcov_read_unsigned (); + + /* Read in the function/callsite profile, and store it in local + data structure. */ + unsigned function_num = gcov_read_unsigned (); + for (unsigned i = 0; i < function_num; i++) { - bb->flags |= BB_ANNOTATED; - return max_count; + Symbol::SymbolStack stack; + const Symbol *symbol = Symbol::ReadSymbol (&stack, gcov_read_counter ()); + afdo_profile_info->sum_all += symbol->total_count (); + map_[symbol->name ()] = symbol; } - else - return 0; + return true; } -/* Annotate auto profile to the control flow graph. */ - -static void -afdo_annotate_cfg (void) +const Symbol *SymbolMap::GetSymbolByInlineStack (const InlineStack &stack) const { - basic_block bb; - gcov_type max_count = ENTRY_BLOCK_PTR->count; + NameSymbolMap::const_iterator iter = map_.find ( + string_map->GetIndexByDecl (stack[stack.size() - 1].func_decl)); + return iter == map_.end() + ? NULL : iter->second->GetSymbol (stack, stack.size() - 1); +} - FOR_EACH_BB (bb) + +/* Member functions for ModuleMap. */ + +bool ModuleMap::Read () +{ + /* Read in the module info. */ + if (gcov_read_unsigned () != GCOV_TAG_AFDO_MODULE_GROUPING) { - bb->count = afdo_get_bb_count (bb); - if (bb->count > max_count) - max_count = bb->count; + inform (0, "Not expected TAG."); + return false; } - if (ENTRY_BLOCK_PTR->count > ENTRY_BLOCK_PTR->next_bb->count) + /* Skip the length of the section. */ + gcov_read_unsigned (); + + /* Read in the file name table. */ + unsigned total_module_num = gcov_read_unsigned (); + for (unsigned i = 0; i < total_module_num; i++) { - ENTRY_BLOCK_PTR->next_bb->count = ENTRY_BLOCK_PTR->count; - ENTRY_BLOCK_PTR->next_bb->flags |= BB_ANNOTATED; + char *name = xstrdup (gcov_read_string ()); + unsigned total_num = 0; + unsigned num_array[6]; + unsigned exported = gcov_read_unsigned (); + unsigned lang = gcov_read_unsigned (); + unsigned ggc_memory = gcov_read_unsigned (); + for (unsigned j = 0; j < 6; j++) + { + num_array[j] = gcov_read_unsigned (); + total_num += num_array[j]; + } + gcov_module_info *module = XCNEWVAR ( + gcov_module_info, + sizeof (gcov_module_info) + sizeof (char *) * total_num); + + std::pair ret = map_.insert( + NameModuleMap::value_type (name, AuxInfo())); + gcc_assert (ret.second); + ret.first->second.second = module; + module->ident = i + 1; + module->lang = lang; + module->ggc_memory = ggc_memory; + module->num_quote_paths = num_array[1]; + module->num_bracket_paths = num_array[2]; + module->num_cpp_defines = num_array[3]; + module->num_cpp_includes = num_array[4]; + module->num_cl_args = num_array[5]; + module->source_filename = name; + module->is_primary = strcmp (name, in_fnames[0]) == 0; + module->flags = module->is_primary ? exported : 1; + for (unsigned j = 0; j < num_array[0]; j++) + ret.first->second.first.push_back (xstrdup (gcov_read_string ())); + for (unsigned j = 0; j < total_num - num_array[0]; j++) + module->string_array[j] = xstrdup (gcov_read_string ()); } - if (ENTRY_BLOCK_PTR->count > EXIT_BLOCK_PTR->prev_bb->count) - { - EXIT_BLOCK_PTR->prev_bb->count = ENTRY_BLOCK_PTR->count; - EXIT_BLOCK_PTR->prev_bb->flags |= BB_ANNOTATED; - } - if (max_count > 0) - { - afdo_calculate_branch_prob (); - counts_to_freqs (); - profile_status = PROFILE_READ; - } - if (flag_value_profile_transformations) - gimple_value_profile_transformations (); + return true; } -extern gcov_working_set_t *gcov_working_sets; - -/* Read profile from profile data file. Write to the module hashmap. */ - static void read_profile (void) { - gcov_unsigned_t i, j, k, file_name_num; - gcov_working_set_t set[128]; - if (gcov_open (auto_profile_file, 1) == 0) { inform (0, "Cannot open profile file %s.", auto_profile_file); @@ -960,142 +628,39 @@ read_profile (void) return; } - /* GCOV_VERSION. */ + /* Skip the version number. */ gcov_read_unsigned (); /* Skip the empty integer. */ gcov_read_unsigned (); - gcc_assert (gcov_read_unsigned () == GCOV_TAG_AFDO_FILE_NAMES); - /* Skip the length of the section. */ - gcov_read_unsigned (); - - /* Read in the file name table. */ - file_name_num = gcov_read_unsigned (); - file_names = (const char **) - xmalloc (sizeof (const char *) * file_name_num); - for (i = 0; i < file_name_num; i++) - file_names[i] = xstrdup (gcov_read_string ()); - - if (gcov_read_unsigned () != GCOV_TAG_AFDO_FUNCTION) + /* StringMap. */ + string_map = StringMap::Create (); + if (string_map == NULL) { - inform (0, "Not expected TAG."); + inform (0, "Cannot read string table."); + flag_auto_profile = 0; return; } - /* Skip the length of the section. */ - gcov_read_unsigned (); - - /* Read in the function/callsite profile, and store it in local - data structure. */ - function_num = gcov_read_unsigned (); - gcov_functions = (struct gcov_function *) - xmalloc (function_num * sizeof (struct gcov_function)); - for (i = 0; i < function_num; i++) + /* SymbolMap. */ + symbol_map = SymbolMap::Create (); + if (symbol_map == NULL) { - gcov_functions[i].name = file_names[gcov_read_unsigned ()]; - gcov_functions[i].file = file_names[gcov_read_unsigned ()]; - gcov_functions[i].total_count = gcov_read_counter (); - gcov_functions[i].entry_count = gcov_read_counter (); - gcov_functions[i].max_count = 0; - gcov_functions[i].stack_num = gcov_read_unsigned (); - gcov_functions[i].stacks = (struct gcov_stack *) - xmalloc (gcov_functions[i].stack_num * sizeof (struct gcov_stack)); - for (j = 0; j < gcov_functions[i].stack_num; j++) - { - gcov_functions[i].stacks[j].func_name = gcov_functions[i].name; - gcov_functions[i].stacks[j].callee_name = NULL; - gcov_functions[i].stacks[j].size = gcov_read_unsigned (); - gcov_functions[i].stacks[j].stack = (struct gcov_callsite_pos *) - xmalloc (gcov_functions[i].stacks[j].size - * sizeof (struct gcov_callsite_pos)); - for (k = 0; k < gcov_functions[i].stacks[j].size; k++) - { - gcov_unsigned_t line, start_line; - gcov_functions[i].stacks[j].stack[k].func = - file_names[gcov_read_unsigned ()]; - gcov_functions[i].stacks[j].stack[k].file = - file_names[gcov_read_unsigned ()]; - line = gcov_read_unsigned (); - start_line = gcov_read_unsigned (); - gcov_functions[i].stacks[j].stack[k].line = - line > start_line ? line - start_line : 0; - gcov_functions[i].stacks[j].stack[k].discr = - gcov_read_unsigned (); - } - gcov_functions[i].stacks[j].count = gcov_read_counter (); - gcov_functions[i].stacks[j].num_inst = gcov_read_counter (); - gcov_functions[i].stacks[j].hist_size = gcov_read_unsigned (); - if (gcov_functions[i].stacks[j].hist_size > 0) - gcov_functions[i].stacks[j].hist = (struct gcov_hist *) - xmalloc (gcov_functions[i].stacks[j].hist_size - * sizeof (struct gcov_hist)); - else - gcov_functions[i].stacks[j].hist = NULL; - for (k = 0; k < gcov_functions[i].stacks[j].hist_size; k++) - { - gcov_functions[i].stacks[j].hist[k].type = - (enum hist_type) gcov_read_unsigned (); - if (gcov_functions[i].stacks[j].hist[k].type == - HIST_TYPE_INDIR_CALL_TOPN) - gcov_functions[i].stacks[j].hist[k].value.func_name = - file_names[gcov_read_counter ()]; - else - gcov_functions[i].stacks[j].hist[k].value.value = - gcov_read_counter (); - gcov_functions[i].stacks[j].hist[k].count = gcov_read_counter (); - } - } + inform (0, "Cannot read function profile."); + flag_auto_profile = 0; + return; } - /* Read in the module info. */ - if (gcov_read_unsigned () != GCOV_TAG_AFDO_MODULE_GROUPING) + /* ModuleMap. */ + module_map = ModuleMap::Create (); + if (module_map == NULL) { - inform (0, "Not expected TAG."); + inform (0, "Cannot read module profile."); + flag_auto_profile = 0; return; } - /* Skip the length of the section. */ - gcov_read_unsigned (); - /* Read in the file name table. */ - total_module_num = gcov_read_unsigned (); - modules = (struct afdo_module *) - xmalloc (total_module_num * sizeof (struct afdo_module)); - for (i = 0; i < total_module_num; i++) - { - unsigned num_strings; - struct afdo_module **slot; - modules[i].name = xstrdup (gcov_read_string ()); - modules[i].ident = i + 1; - /* exported flag. */ - modules[i].exported = gcov_read_unsigned (); - modules[i].lang = gcov_read_unsigned (); - modules[i].ggc_memory = gcov_read_unsigned (); - /* aux_module and 5 options. */ - modules[i].num_aux_modules = gcov_read_unsigned (); - modules[i].num_quote_paths = gcov_read_unsigned (); - modules[i].num_bracket_paths = gcov_read_unsigned (); - modules[i].num_cpp_defines = gcov_read_unsigned (); - modules[i].num_cpp_includes = gcov_read_unsigned (); - modules[i].num_cl_args = gcov_read_unsigned (); - num_strings = modules[i].num_aux_modules - + modules[i].num_quote_paths - + modules[i].num_bracket_paths - + modules[i].num_cpp_defines - + modules[i].num_cpp_includes - + modules[i].num_cl_args; - modules[i].strings = (char **) - xmalloc (num_strings * sizeof (char *)); - for (j = 0; j < num_strings; j++) - modules[i].strings[j] = xstrdup (gcov_read_string ()); - slot = (struct afdo_module **) - htab_find_slot (module_htab, &modules[i], INSERT); - if (!*slot) - *slot = &modules[i]; - else - gcc_unreachable (); - } - /* Read in the working set. */ if (gcov_read_unsigned () != GCOV_TAG_AFDO_WORKING_SET) { @@ -1105,7 +670,8 @@ read_profile (void) /* Skip the length of the section. */ gcov_read_unsigned (); - for (i = 0; i < 128; i++) + gcov_working_set_t set[128]; + for (unsigned i = 0; i < 128; i++) { set[i].num_counters = gcov_read_unsigned (); set[i].min_counter = gcov_read_counter (); @@ -1113,158 +679,155 @@ read_profile (void) add_working_set (set); } -/* Process the profile data and build the function/stack - hash maps. */ +/* Read in the auxiliary modules for the current primary module. */ -void -process_auto_profile (void) +static void +read_aux_modules (void) { - unsigned i; + gcov_module_info *module = module_map->GetModule (in_fnames[0]); + if (module == NULL) + return; - afdo_read_bfd_names (); - for (i = 0; i < function_num; i++) + const StringVector *aux_modules = module_map->GetAuxModules (in_fnames[0]); + unsigned num_aux_modules = aux_modules ? aux_modules->size() : 0; + + module_infos = XCNEWVEC (gcov_module_info *, num_aux_modules + 1); + module_infos[0] = module; + primary_module_id = module->ident; + if (aux_modules == NULL) + return; + unsigned curr_module = 1; + for (StringVector::const_iterator iter = aux_modules->begin(); + iter != aux_modules->end(); ++iter) { - struct gcov_function **func_slot = (struct gcov_function **) - htab_find_slot (function_htab, gcov_functions + i, INSERT); - if (*func_slot) + gcov_module_info *aux_module = + module_map->GetModule (*iter); + if (aux_module == module) + continue; + if (aux_module == NULL) { - (*func_slot)->entry_count += gcov_functions[i].entry_count; - (*func_slot)->total_count += gcov_functions[i].total_count; - afdo_profile_info->sum_all += (*func_slot)->total_count; + inform (0, "aux module %s cannot be found.", *iter); + continue; } - else - *func_slot = gcov_functions + i; - } - - for (i = 0; i < function_num; i++) - { - unsigned j; - struct gcov_function *func = gcov_functions + i; - for (j = 0; j < func->stack_num; j++) + if ((aux_module->lang & GCOV_MODULE_LANG_MASK) != + (module->lang & GCOV_MODULE_LANG_MASK)) { - unsigned k; - unsigned stack_size = func->stacks[j].size; - gcov_type count = func->stacks[j].count; - struct gcov_stack **stack_slot = (struct gcov_stack **) - htab_find_slot (stack_htab, func->stacks + j, INSERT); - if (func->stacks[j].num_inst && count > afdo_profile_info->sum_max) - afdo_profile_info->sum_max = count / func->stacks[j].num_inst; - if (*stack_slot) - { - (*stack_slot)->count += count; - if ((*stack_slot)->num_inst < func->stacks[j].num_inst) - (*stack_slot)->num_inst = func->stacks[j].num_inst; - } - else - *stack_slot = func->stacks + j; - for (k = 1; k < stack_size; k++) - { - struct gcov_stack *new_stack = (struct gcov_stack *) - xmalloc (sizeof (struct gcov_stack)); - new_stack->func_name = func->stacks[j].func_name; - new_stack->callee_name = - func->stacks[j].stack[stack_size - k - 1].func; - new_stack->stack = func->stacks[j].stack + stack_size - k; - new_stack->size = k; - new_stack->num_inst = 0; - new_stack->count = 0; - new_stack->max_count = 0; - new_stack->hist_size = 0; - new_stack->hist = NULL; - stack_slot = (struct gcov_stack **) - htab_find_slot (stack_htab, new_stack, INSERT); - if (!*stack_slot) - *stack_slot = new_stack; - else - free (new_stack); - (*stack_slot)->count += count; - if ((*stack_slot)->max_count < count) - (*stack_slot)->max_count = count; - } + inform (0, "Not importing %s: source language" + " different from primary module's source language", *iter); + continue; } + if ((aux_module->lang & GCOV_MODULE_ASM_STMTS) + && flag_ripa_disallow_asm_modules) + { + if (flag_opt_info) + inform (0, "Not importing %s: contains " + "assembler statements", *iter); + continue; + } + if (incompatible_cl_args (module, aux_module)) + { + if (flag_opt_info) + inform (0, "Not importing %s: command-line" + " arguments not compatible with primary module", *iter); + continue; + } + module_infos[curr_module++] = aux_module; + add_input_filename (*iter); } } -/* Create the hash tables, and read the profile from the profile data - file. */ +/* From AutoFDO profiles, find values inside STMT for that we want to measure + histograms for indirect-call optimization. */ -void -init_auto_profile (void) +static void +afdo_indirect_call (gimple stmt, const TargetMap &map) { - if (auto_profile_file == NULL) - auto_profile_file = DEFAULT_AUTO_PROFILE_FILE; + tree callee; - /* Initialize the function hash table. */ - function_htab = htab_create_alloc ((size_t) SP_HTAB_INIT_SIZE, - afdo_function_hash, - afdo_function_eq, - 0, - xcalloc, - free); - /* Initialize the stack hash table. */ - stack_htab = htab_create_alloc ((size_t) SP_HTAB_INIT_SIZE, - afdo_stack_hash, - afdo_stack_eq, - 0, - xcalloc, - free); - /* Initialize the bfd name mapping table. */ - bfd_name_htab = htab_create_alloc ((size_t) SP_HTAB_INIT_SIZE, - afdo_bfd_name_hash, - afdo_bfd_name_eq, - afdo_bfd_name_del, - xcalloc, - free); - /* Initialize the module hash table. */ - module_htab = htab_create_alloc ((size_t) SP_HTAB_INIT_SIZE, - afdo_module_hash, - afdo_module_eq, - 0, - xcalloc, - free); + if (map.size() == 0 || gimple_code (stmt) != GIMPLE_CALL + || gimple_call_fndecl (stmt) != NULL_TREE) + return; - afdo_profile_info = (struct gcov_ctr_summary *) - xcalloc (1, sizeof (struct gcov_ctr_summary)); - afdo_profile_info->runs = 1; - afdo_profile_info->sum_max = 0; - afdo_profile_info->sum_all = 0; + callee = gimple_call_fn (stmt); - /* Read the profile from the profile file. */ - read_profile (); + histogram_value hist = gimple_alloc_histogram_value ( + cfun, HIST_TYPE_INDIR_CALL_TOPN, stmt, callee); + hist->n_counters = (GCOV_ICALL_TOPN_VAL << 2) + 1; + hist->hvalue.counters = XNEWVEC (gcov_type, hist->n_counters); + gimple_add_histogram_value (cfun, stmt, hist); - if (flag_dyn_ipa) - read_aux_modules (); + gcov_type total = 0; + TargetMap::const_iterator max_iter1 = map.end(); + TargetMap::const_iterator max_iter2 = map.end(); + + for (TargetMap::const_iterator iter = map.begin(); iter != map.end(); iter++) + { + total += iter->second; + if (max_iter1 == map.end() || max_iter1->second < iter->second) + { + max_iter2 = max_iter1; + max_iter1 = iter; + } + else if (max_iter2 == map.end() || max_iter2->second < iter->second) + max_iter2 = iter; + } + + hist->hvalue.counters[0] = total; + hist->hvalue.counters[1] = + (unsigned long long) string_map->GetName (max_iter1->first); + hist->hvalue.counters[2] = max_iter1->second; + if (max_iter2 != map.end()) + { + hist->hvalue.counters[3] = + (unsigned long long) string_map->GetName (max_iter2->first); + hist->hvalue.counters[4] = max_iter2->second; + } + else + { + hist->hvalue.counters[3] = 0; + hist->hvalue.counters[4] = 0; + } } -/* Free the resources. */ +/* From AutoFDO profiles, find values inside STMT for that we want to measure + histograms and adds them to list VALUES. */ -void -end_auto_profile (void) +static void +afdo_vpt (gimple stmt, const TargetMap &map) { - unsigned i, j; + afdo_indirect_call (stmt, map); +} - for (i = 0; i < function_num; i++) +/* For a given BB, return its execution count, and annotate value profile + on statements. */ + +static gcov_type +afdo_get_bb_count (basic_block bb) +{ + gimple_stmt_iterator gsi; + gcov_type max_count = 0; + bool has_annotated = false; + + for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) { - for (j = 0; j < gcov_functions[i].stack_num; ++j) + ProfileInfo info; + gimple stmt = gsi_stmt (gsi); + if (symbol_map->GetProfileInfo (stmt, &info)) { - if (gcov_functions[i].stacks[j].hist_size > 0) - free (gcov_functions[i].stacks[j].hist); - free (gcov_functions[i].stacks[j].stack); + if (info.count > max_count) + max_count = info.count; + has_annotated = true; + if (info.target_map.size() > 0) + afdo_vpt (stmt, info.target_map); } - free (gcov_functions[i].stacks); } - free (gcov_functions); - - for (i = 0; i < total_module_num; i++) - free (modules[i].strings); - free (modules); - free (afdo_profile_info); - free (file_names); - htab_delete (function_htab); - htab_delete (stack_htab); - htab_delete (bfd_name_htab); - htab_delete (module_htab); - profile_info = NULL; + if (has_annotated) + { + bb->flags |= BB_ANNOTATED; + return max_count; + } + else + return 0; } /* BB1 and BB2 are in an equivalent class iff: @@ -1566,7 +1129,7 @@ afdo_propagate (void) /* Propagate counts on control flow graph and calculate branch probabilities. */ -void +static void afdo_calculate_branch_prob (void) { basic_block bb; @@ -1623,25 +1186,42 @@ afdo_calculate_branch_prob (void) free_dominance_info (CDI_POST_DOMINATORS); } -/* Returns TRUE if EDGE is hot enough to be inlined early. */ +/* Annotate auto profile to the control flow graph. */ -bool -afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge *edge) +static void +afdo_annotate_cfg (void) { - gcov_type count, max_count; - if (get_callsite_count (edge, &count, &max_count)) + basic_block bb; + const Symbol *symbol = symbol_map->GetSymbolByDecl (current_function_decl); + if (symbol == NULL) + return; + ENTRY_BLOCK_PTR->count = symbol->head_count (); + gcov_type max_count = ENTRY_BLOCK_PTR->count; + + FOR_EACH_BB (bb) { - bool is_hot; - const struct gcov_ctr_summary *saved_profile_info = profile_info; - /* At earling inline stage, profile_info is not set yet. We need to - temporarily set it to afdo_profile_info to calculate hotness. */ - profile_info = afdo_profile_info; - is_hot = maybe_hot_count_p (NULL, count); - profile_info = saved_profile_info; - return is_hot; + bb->count = afdo_get_bb_count (bb); + if (bb->count > max_count) + max_count = bb->count; } - else - return false; + if (ENTRY_BLOCK_PTR->count > ENTRY_BLOCK_PTR->next_bb->count) + { + ENTRY_BLOCK_PTR->next_bb->count = ENTRY_BLOCK_PTR->count; + ENTRY_BLOCK_PTR->next_bb->flags |= BB_ANNOTATED; + } + if (ENTRY_BLOCK_PTR->count > EXIT_BLOCK_PTR->prev_bb->count) + { + EXIT_BLOCK_PTR->prev_bb->count = ENTRY_BLOCK_PTR->count; + EXIT_BLOCK_PTR->prev_bb->flags |= BB_ANNOTATED; + } + if (max_count > 0) + { + afdo_calculate_branch_prob (); + counts_to_freqs (); + profile_status = PROFILE_READ; + } + if (flag_value_profile_transformations) + gimple_value_profile_transformations (); } /* Use AutoFDO profile to annoate the control flow graph. @@ -1693,6 +1273,59 @@ gate_auto_profile_ipa (void) return flag_auto_profile; } +/* Read the profile from the profile data file. */ + +void +init_auto_profile (void) +{ + if (auto_profile_file == NULL) + auto_profile_file = DEFAULT_AUTO_PROFILE_FILE; + + afdo_profile_info = (struct gcov_ctr_summary *) + xcalloc (1, sizeof (struct gcov_ctr_summary)); + afdo_profile_info->runs = 1; + afdo_profile_info->sum_max = 0; + afdo_profile_info->sum_all = 0; + + /* Read the profile from the profile file. */ + read_profile (); + + if (flag_dyn_ipa) + read_aux_modules (); +} + +/* Free the resources. */ + +void +end_auto_profile (void) +{ + delete symbol_map; + delete string_map; + delete module_map; + profile_info = NULL; +} + +/* Returns TRUE if EDGE is hot enough to be inlined early. */ + +bool +afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge *edge) +{ + gcov_type count = symbol_map->GetCallsiteTotalCount (edge); + if (count > 0) + { + bool is_hot; + const struct gcov_ctr_summary *saved_profile_info = profile_info; + /* At earling inline stage, profile_info is not set yet. We need to + temporarily set it to afdo_profile_info to calculate hotness. */ + profile_info = afdo_profile_info; + is_hot = maybe_hot_count_p (NULL, count); + profile_info = saved_profile_info; + return is_hot; + } + else + return false; +} + struct simple_ipa_opt_pass pass_ipa_auto_profile = { { Index: gcc/auto-profile.h =================================================================== --- gcc/auto-profile.h (revision 201022) +++ gcc/auto-profile.h (working copy) @@ -24,17 +24,7 @@ along with GCC; see the file COPYING3. If not see /* Read, process, finalize AutoFDO data structures. */ extern void init_auto_profile (void); extern void end_auto_profile (void); -extern void process_auto_profile (void); -/* Annotate function's count and total count. */ -extern void afdo_set_current_function_count (void); - -/* Add the assembly_name to bfd_name mapping. */ -extern void afdo_add_bfd_name_mapping (const char *, const char *); - -/* Calculate branch probability in both AutoFDO pass and after inlining. */ -extern void afdo_calculate_branch_prob (void); - /* Returns TRUE if EDGE is hot enough to be inlined early. */ extern bool afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge *); Index: gcc/predict.c =================================================================== --- gcc/predict.c (revision 201022) +++ gcc/predict.c (working copy) @@ -56,7 +56,6 @@ along with GCC; see the file COPYING3. If not see #include "tree-scalar-evolution.h" #include "cfgloop.h" #include "pointer-set.h" -#include "auto-profile.h" /* real constants: 0, 1, 1-1/REG_BR_PROB_BASE, REG_BR_PROB_BASE, 1/REG_BR_PROB_BASE, 0.5, BB_FREQ_MAX. */ @@ -2939,26 +2938,13 @@ rebuild_frequencies (void) timevar_push (TV_REBUILD_FREQUENCIES); if (profile_status == PROFILE_GUESSED) { - /* In AutoFDO it is possible that some basic blocks will get - non-zero counts after function inlining. In this case, we - will use profile information to estimated the frequency. */ - if (flag_auto_profile && counts_to_freqs ()) - { - afdo_calculate_branch_prob (); - counts_to_freqs(); - profile_status = PROFILE_READ; - compute_function_frequency (); - } - else - { - loop_optimizer_init (0); - add_noreturn_fake_exit_edges (); - mark_irreducible_loops (); - connect_infinite_loops_to_exit (); - estimate_bb_frequencies (); - remove_fake_exit_edges (); - loop_optimizer_finalize (); - } + loop_optimizer_init (0); + add_noreturn_fake_exit_edges (); + mark_irreducible_loops (); + connect_infinite_loops_to_exit (); + estimate_bb_frequencies (); + remove_fake_exit_edges (); + loop_optimizer_finalize (); } else if (profile_status == PROFILE_READ) counts_to_freqs ();