cgraph.h (cgraph_node): Add profile_id.
* cgraph.h (cgraph_node): Add profile_id. * value-prof.c (cgraph_node_map): Turn into pointer_map. (init_node_map): Rewrite to handle hashes increas of incremental IDs. (del_node_map): Update. (find_func_by_funcdef_no): Replace by ... (find_func_by_profile_id): ... this one. (gimple_ic_transform): Do not remove useful histograms when speculation is not done; dump info when indirect call removal can happen at LTO. * value-prof.h (find_func_by_profile_id, gimple_ic): Declare. * gcov-io.h (__gcov_indirect_call_profiler): Replace by ... (__gcov_indirect_call_profiler_v2): .. this one. * profile.h (init_node_map): Update. * coverage.c (coverage_compute_profile_id): New function. * coverage.h (coverage_compute_profile_id): Declare. * tree-profile.c (init_ic_make_global_vars): Make __gcov_indirect_call_callee and __gcov_indirect_call_counters global. (gimple_init_edge_profiler): Update prototype of __gcov_indirect_call_profiler. (gimple_gen_ic_func_profiler): Simplify. (tree_profiling): Use init_node_map From-SVN: r201634
This commit is contained in:
Jan Hubicka
Jan Hubicka
parent
36849c21cc
commit
2fa3d31bd6
11 changed files with 188 additions and 65 deletions
|
|
@ -1,3 +1,28 @@
|
|||
2013-08-09 Jan Hubicka <jh@suse.cz>
|
||||
|
||||
* cgraph.h (cgraph_node): Add profile_id.
|
||||
* value-prof.c (cgraph_node_map): Turn into pointer_map.
|
||||
(init_node_map): Rewrite to handle hashes increas of incremental
|
||||
IDs.
|
||||
(del_node_map): Update.
|
||||
(find_func_by_funcdef_no): Replace by ...
|
||||
(find_func_by_profile_id): ... this one.
|
||||
(gimple_ic_transform): Do not remove useful histograms when
|
||||
speculation is not done; dump info when indirect call removal
|
||||
can happen at LTO.
|
||||
* value-prof.h (find_func_by_profile_id, gimple_ic): Declare.
|
||||
* gcov-io.h (__gcov_indirect_call_profiler): Replace by ...
|
||||
(__gcov_indirect_call_profiler_v2): .. this one.
|
||||
* profile.h (init_node_map): Update.
|
||||
* coverage.c (coverage_compute_profile_id): New function.
|
||||
* coverage.h (coverage_compute_profile_id): Declare.
|
||||
* tree-profile.c (init_ic_make_global_vars): Make
|
||||
__gcov_indirect_call_callee and __gcov_indirect_call_counters global.
|
||||
(gimple_init_edge_profiler): Update prototype of
|
||||
__gcov_indirect_call_profiler.
|
||||
(gimple_gen_ic_func_profiler): Simplify.
|
||||
(tree_profiling): Use init_node_map
|
||||
|
||||
2013-08-09 Jan Hubicka <jh@suse.cz>
|
||||
|
||||
* cgraphbuild.c (cgraph_rebuild_references): Rebuild only non-speculative
|
||||
|
|
|
|||
|
|
@ -300,6 +300,8 @@ struct GTY(()) cgraph_node {
|
|||
int count_materialization_scale;
|
||||
/* Unique id of the node. */
|
||||
int uid;
|
||||
/* ID assigned by the profiling. */
|
||||
unsigned int profile_id;
|
||||
|
||||
/* Set when decl is an abstract function pointed to by the
|
||||
ABSTRACT_DECL_ORIGIN of a reachable function. */
|
||||
|
|
|
|||
|
|
@ -539,6 +539,28 @@ coverage_compute_lineno_checksum (void)
|
|||
return chksum;
|
||||
}
|
||||
|
||||
/* Compute profile ID. This is better to be unique in whole program. */
|
||||
|
||||
unsigned
|
||||
coverage_compute_profile_id (struct cgraph_node *n)
|
||||
{
|
||||
expanded_location xloc
|
||||
= expand_location (DECL_SOURCE_LOCATION (n->symbol.decl));
|
||||
unsigned chksum = xloc.line;
|
||||
|
||||
chksum = coverage_checksum_string (chksum, xloc.file);
|
||||
chksum = coverage_checksum_string
|
||||
(chksum, IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (n->symbol.decl)));
|
||||
if (first_global_object_name)
|
||||
chksum = coverage_checksum_string
|
||||
(chksum, first_global_object_name);
|
||||
chksum = coverage_checksum_string
|
||||
(chksum, aux_base_name);
|
||||
|
||||
/* Non-negative integers are hopefully small enough to fit in all targets. */
|
||||
return chksum & 0x7fffffff;
|
||||
}
|
||||
|
||||
/* Compute cfg checksum for the current function.
|
||||
The checksum is calculated carefully so that
|
||||
source code changes that doesn't affect the control flow graph
|
||||
|
|
|
|||
|
|
@ -35,6 +35,9 @@ extern void coverage_end_function (unsigned, unsigned);
|
|||
/* Compute the control flow checksum for the current function. */
|
||||
extern unsigned coverage_compute_cfg_checksum (void);
|
||||
|
||||
/* Compute the profile id of function N. */
|
||||
extern unsigned coverage_compute_profile_id (struct cgraph_node *n);
|
||||
|
||||
/* Compute the line number checksum for the current function. */
|
||||
extern unsigned coverage_compute_lineno_checksum (void);
|
||||
|
||||
|
|
|
|||
|
|
@ -515,7 +515,7 @@ extern void __gcov_merge_ior (gcov_type *, unsigned) ATTRIBUTE_HIDDEN;
|
|||
extern void __gcov_interval_profiler (gcov_type *, gcov_type, int, unsigned);
|
||||
extern void __gcov_pow2_profiler (gcov_type *, gcov_type);
|
||||
extern void __gcov_one_value_profiler (gcov_type *, gcov_type);
|
||||
extern void __gcov_indirect_call_profiler (gcov_type *, gcov_type, void *, void *);
|
||||
extern void __gcov_indirect_call_profiler_v2 (gcov_type, void *);
|
||||
extern void __gcov_average_profiler (gcov_type *, gcov_type);
|
||||
extern void __gcov_ior_profiler (gcov_type *, gcov_type);
|
||||
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ extern void mcf_smooth_cfg (void);
|
|||
|
||||
extern gcov_type sum_edge_counts (vec<edge, va_gc> *edges);
|
||||
|
||||
extern void init_node_map (void);
|
||||
extern void init_node_map (bool);
|
||||
extern void del_node_map (void);
|
||||
|
||||
extern void get_working_sets (void);
|
||||
|
|
|
|||
|
|
@ -57,8 +57,8 @@ static GTY(()) tree ptr_void;
|
|||
/* Do initialization work for the edge profiler. */
|
||||
|
||||
/* Add code:
|
||||
static gcov* __gcov_indirect_call_counters; // pointer to actual counter
|
||||
static void* __gcov_indirect_call_callee; // actual callee address
|
||||
__thread gcov* __gcov_indirect_call_counters; // pointer to actual counter
|
||||
__thread void* __gcov_indirect_call_callee; // actual callee address
|
||||
*/
|
||||
static void
|
||||
init_ic_make_global_vars (void)
|
||||
|
|
@ -72,7 +72,8 @@ init_ic_make_global_vars (void)
|
|||
get_identifier ("__gcov_indirect_call_callee"),
|
||||
ptr_void);
|
||||
TREE_STATIC (ic_void_ptr_var) = 1;
|
||||
TREE_PUBLIC (ic_void_ptr_var) = 0;
|
||||
TREE_PUBLIC (ic_void_ptr_var) = 1;
|
||||
DECL_EXTERNAL (ic_void_ptr_var) = 1;
|
||||
DECL_ARTIFICIAL (ic_void_ptr_var) = 1;
|
||||
DECL_INITIAL (ic_void_ptr_var) = NULL;
|
||||
if (targetm.have_tls)
|
||||
|
|
@ -87,7 +88,8 @@ init_ic_make_global_vars (void)
|
|||
get_identifier ("__gcov_indirect_call_counters"),
|
||||
gcov_type_ptr);
|
||||
TREE_STATIC (ic_gcov_type_ptr_var) = 1;
|
||||
TREE_PUBLIC (ic_gcov_type_ptr_var) = 0;
|
||||
TREE_PUBLIC (ic_gcov_type_ptr_var) = 1;
|
||||
DECL_EXTERNAL (ic_gcov_type_ptr_var) = 1;
|
||||
DECL_ARTIFICIAL (ic_gcov_type_ptr_var) = 1;
|
||||
DECL_INITIAL (ic_gcov_type_ptr_var) = NULL;
|
||||
if (targetm.have_tls)
|
||||
|
|
@ -155,14 +157,14 @@ gimple_init_edge_profiler (void)
|
|||
|
||||
init_ic_make_global_vars ();
|
||||
|
||||
/* void (*) (gcov_type *, gcov_type, void *, void *) */
|
||||
/* void (*) (gcov_type, void *) */
|
||||
ic_profiler_fn_type
|
||||
= build_function_type_list (void_type_node,
|
||||
gcov_type_ptr, gcov_type_node,
|
||||
gcov_type_node,
|
||||
ptr_void,
|
||||
ptr_void, NULL_TREE);
|
||||
NULL_TREE);
|
||||
tree_indirect_call_profiler_fn
|
||||
= build_fn_decl ("__gcov_indirect_call_profiler",
|
||||
= build_fn_decl ("__gcov_indirect_call_profiler_v2",
|
||||
ic_profiler_fn_type);
|
||||
TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1;
|
||||
DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)
|
||||
|
|
@ -352,7 +354,7 @@ gimple_gen_ic_func_profiler (void)
|
|||
struct cgraph_node * c_node = cgraph_get_node (current_function_decl);
|
||||
gimple_stmt_iterator gsi;
|
||||
gimple stmt1, stmt2;
|
||||
tree tree_uid, cur_func, counter_ptr, ptr_var, void0;
|
||||
tree tree_uid, cur_func, void0;
|
||||
|
||||
if (cgraph_only_called_directly_p (c_node))
|
||||
return;
|
||||
|
|
@ -361,27 +363,20 @@ gimple_gen_ic_func_profiler (void)
|
|||
|
||||
/* Insert code:
|
||||
|
||||
stmt1: __gcov_indirect_call_profiler (__gcov_indirect_call_counters,
|
||||
current_function_funcdef_no,
|
||||
¤t_function_decl,
|
||||
__gcov_indirect_call_callee);
|
||||
stmt1: __gcov_indirect_call_profiler_v2 (profile_id,
|
||||
¤t_function_decl)
|
||||
*/
|
||||
gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR));
|
||||
gsi = gsi_after_labels (split_edge (single_succ_edge (ENTRY_BLOCK_PTR)));
|
||||
|
||||
cur_func = force_gimple_operand_gsi (&gsi,
|
||||
build_addr (current_function_decl,
|
||||
current_function_decl),
|
||||
true, NULL_TREE,
|
||||
true, GSI_SAME_STMT);
|
||||
counter_ptr = force_gimple_operand_gsi (&gsi, ic_gcov_type_ptr_var,
|
||||
true, NULL_TREE, true,
|
||||
GSI_SAME_STMT);
|
||||
ptr_var = force_gimple_operand_gsi (&gsi, ic_void_ptr_var,
|
||||
true, NULL_TREE, true,
|
||||
GSI_SAME_STMT);
|
||||
tree_uid = build_int_cst (gcov_type_node, current_function_funcdef_no);
|
||||
stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 4,
|
||||
counter_ptr, tree_uid, cur_func, ptr_var);
|
||||
tree_uid = build_int_cst
|
||||
(gcov_type_node, cgraph_get_node (current_function_decl)->profile_id);
|
||||
stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2,
|
||||
tree_uid, cur_func);
|
||||
gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
|
||||
|
||||
/* Set __gcov_indirect_call_callee to 0,
|
||||
|
|
@ -461,7 +456,7 @@ tree_profiling (void)
|
|||
cgraphunit.c:ipa_passes(). */
|
||||
gcc_assert (cgraph_state == CGRAPH_STATE_IPA_SSA);
|
||||
|
||||
init_node_map();
|
||||
init_node_map (true);
|
||||
|
||||
FOR_EACH_DEFINED_FUNCTION (node)
|
||||
{
|
||||
|
|
|
|||
119
gcc/value-prof.c
119
gcc/value-prof.c
|
|
@ -1173,24 +1173,67 @@ gimple_mod_subtract_transform (gimple_stmt_iterator *si)
|
|||
return true;
|
||||
}
|
||||
|
||||
static vec<cgraph_node_ptr> cgraph_node_map
|
||||
= vNULL;
|
||||
static pointer_map_t *cgraph_node_map;
|
||||
|
||||
/* Initialize map from FUNCDEF_NO to CGRAPH_NODE. */
|
||||
/* Initialize map from PROFILE_ID to CGRAPH_NODE.
|
||||
When LOCAL is true, the PROFILE_IDs are computed. when it is false we assume
|
||||
that the PROFILE_IDs was already assigned. */
|
||||
|
||||
void
|
||||
init_node_map (void)
|
||||
init_node_map (bool local)
|
||||
{
|
||||
struct cgraph_node *n;
|
||||
cgraph_node_map = pointer_map_create ();
|
||||
|
||||
if (get_last_funcdef_no ())
|
||||
cgraph_node_map.safe_grow_cleared (get_last_funcdef_no ());
|
||||
|
||||
FOR_EACH_FUNCTION (n)
|
||||
{
|
||||
if (DECL_STRUCT_FUNCTION (n->symbol.decl))
|
||||
cgraph_node_map[DECL_STRUCT_FUNCTION (n->symbol.decl)->funcdef_no] = n;
|
||||
}
|
||||
FOR_EACH_DEFINED_FUNCTION (n)
|
||||
if (cgraph_function_with_gimple_body_p (n)
|
||||
&& !cgraph_only_called_directly_p (n))
|
||||
{
|
||||
void **val;
|
||||
if (local)
|
||||
{
|
||||
n->profile_id = coverage_compute_profile_id (n);
|
||||
while ((val = pointer_map_contains (cgraph_node_map,
|
||||
(void *)(size_t)n->profile_id))
|
||||
|| !n->profile_id)
|
||||
{
|
||||
if (dump_file)
|
||||
fprintf (dump_file, "Local profile-id %i conflict"
|
||||
" with nodes %s/%i %s/%i\n",
|
||||
n->profile_id,
|
||||
cgraph_node_name (n),
|
||||
n->symbol.order,
|
||||
symtab_node_name (*(symtab_node*)val),
|
||||
(*(symtab_node *)val)->symbol.order);
|
||||
n->profile_id = (n->profile_id + 1) & 0x7fffffff;
|
||||
}
|
||||
}
|
||||
else if (!n->profile_id)
|
||||
{
|
||||
if (dump_file)
|
||||
fprintf (dump_file,
|
||||
"Node %s/%i has no profile-id"
|
||||
" (profile feedback missing?)\n",
|
||||
cgraph_node_name (n),
|
||||
n->symbol.order);
|
||||
continue;
|
||||
}
|
||||
else if ((val = pointer_map_contains (cgraph_node_map,
|
||||
(void *)(size_t)n->profile_id)))
|
||||
{
|
||||
if (dump_file)
|
||||
fprintf (dump_file,
|
||||
"Node %s/%i has IP profile-id %i conflict. "
|
||||
"Giving up.\n",
|
||||
cgraph_node_name (n),
|
||||
n->symbol.order,
|
||||
n->profile_id);
|
||||
*val = NULL;
|
||||
continue;
|
||||
}
|
||||
*pointer_map_insert (cgraph_node_map,
|
||||
(void *)(size_t)n->profile_id) = (void *)n;
|
||||
}
|
||||
}
|
||||
|
||||
/* Delete the CGRAPH_NODE_MAP. */
|
||||
|
|
@ -1198,27 +1241,20 @@ init_node_map (void)
|
|||
void
|
||||
del_node_map (void)
|
||||
{
|
||||
cgraph_node_map.release ();
|
||||
pointer_map_destroy (cgraph_node_map);
|
||||
}
|
||||
|
||||
/* Return cgraph node for function with pid */
|
||||
|
||||
static inline struct cgraph_node*
|
||||
find_func_by_funcdef_no (int func_id)
|
||||
struct cgraph_node*
|
||||
find_func_by_profile_id (int profile_id)
|
||||
{
|
||||
int max_id = get_last_funcdef_no ();
|
||||
if (func_id >= max_id || cgraph_node_map[func_id] == NULL)
|
||||
{
|
||||
if (flag_profile_correction)
|
||||
inform (DECL_SOURCE_LOCATION (current_function_decl),
|
||||
"Inconsistent profile: indirect call target (%d) does not exist", func_id);
|
||||
else
|
||||
error ("Inconsistent profile: indirect call target (%d) does not exist", func_id);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return cgraph_node_map[func_id];
|
||||
void **val = pointer_map_contains (cgraph_node_map,
|
||||
(void *)(size_t)profile_id);
|
||||
if (val)
|
||||
return (struct cgraph_node *)*val;
|
||||
else
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Perform sanity check on the indirect call target. Due to race conditions,
|
||||
|
|
@ -1415,10 +1451,12 @@ gimple_ic_transform (gimple_stmt_iterator *gsi)
|
|||
val = histogram->hvalue.counters [0];
|
||||
count = histogram->hvalue.counters [1];
|
||||
all = histogram->hvalue.counters [2];
|
||||
gimple_remove_histogram_value (cfun, stmt, histogram);
|
||||
|
||||
if (4 * count <= 3 * all)
|
||||
return false;
|
||||
{
|
||||
gimple_remove_histogram_value (cfun, stmt, histogram);
|
||||
return false;
|
||||
}
|
||||
|
||||
bb_all = gimple_bb (stmt)->count;
|
||||
/* The order of CHECK_COUNTER calls is important -
|
||||
|
|
@ -1426,16 +1464,31 @@ gimple_ic_transform (gimple_stmt_iterator *gsi)
|
|||
and we want to make count <= all <= bb_all. */
|
||||
if ( check_counter (stmt, "ic", &all, &bb_all, bb_all)
|
||||
|| check_counter (stmt, "ic", &count, &all, all))
|
||||
return false;
|
||||
{
|
||||
gimple_remove_histogram_value (cfun, stmt, histogram);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (all > 0)
|
||||
prob = GCOV_COMPUTE_SCALE (count, all);
|
||||
else
|
||||
prob = 0;
|
||||
direct_call = find_func_by_funcdef_no ((int)val);
|
||||
direct_call = find_func_by_profile_id ((int)val);
|
||||
|
||||
if (direct_call == NULL)
|
||||
return false;
|
||||
{
|
||||
if (val)
|
||||
{
|
||||
if (dump_file)
|
||||
{
|
||||
fprintf (dump_file, "Indirect call -> direct call from other module");
|
||||
print_generic_expr (dump_file, gimple_call_fn (stmt), TDF_SLIM);
|
||||
fprintf (dump_file, "=> %i (will resolve only with LTO)\n", (int)val);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
gimple_remove_histogram_value (cfun, stmt, histogram);
|
||||
|
||||
if (!check_ic_target (stmt, direct_call))
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -103,6 +103,8 @@ extern void gimple_gen_average_profiler (histogram_value, unsigned, unsigned);
|
|||
extern void gimple_gen_ior_profiler (histogram_value, unsigned, unsigned);
|
||||
extern void stream_out_histogram_value (struct output_block *, histogram_value);
|
||||
extern void stream_in_histogram_value (struct lto_input_block *, gimple);
|
||||
extern struct cgraph_node* find_func_by_profile_id (int func_id);
|
||||
|
||||
|
||||
/* In profile.c. */
|
||||
extern void init_branch_prob (void);
|
||||
|
|
|
|||
|
|
@ -1,3 +1,10 @@
|
|||
2013-08-06 Jan Hubicka <jh@suse.cz>
|
||||
|
||||
* libgcov.c (__gcov_indirect_call_callee,
|
||||
__gcov_indirect_call_counters): New global vars.
|
||||
(__gcov_indirect_call_profiler): replace by ...
|
||||
(__gcov_indirect_call_profiler_v2) ... this one.
|
||||
|
||||
2013-08-06 Caroline Tice <cmtice@google.com>
|
||||
|
||||
* config.host (extra_parts): Add vtv_start.o, vtv_end.o
|
||||
|
|
|
|||
|
|
@ -1121,6 +1121,21 @@ __gcov_one_value_profiler (gcov_type *counters, gcov_type value)
|
|||
|
||||
#ifdef L_gcov_indirect_call_profiler
|
||||
|
||||
/* These two variables are used to actually track caller and callee. Keep
|
||||
them in TLS memory so races are not common (they are written to often).
|
||||
The variables are set directly by GCC instrumented code, so declaration
|
||||
here must match one in tree-profile.c */
|
||||
|
||||
#ifdef HAVE_CC_TLS
|
||||
__thread
|
||||
#endif
|
||||
void * __gcov_indirect_call_callee;
|
||||
#ifdef HAVE_CC_TLS
|
||||
__thread
|
||||
#endif
|
||||
gcov_type * __gcov_indirect_call_counters;
|
||||
|
||||
|
||||
/* By default, the C++ compiler will use function addresses in the
|
||||
vtable entries. Setting TARGET_VTABLE_USES_DESCRIPTORS to nonzero
|
||||
tells the compiler to use function descriptors instead. The value
|
||||
|
|
@ -1140,16 +1155,15 @@ __gcov_one_value_profiler (gcov_type *counters, gcov_type value)
|
|||
|
||||
/* Tries to determine the most common value among its inputs. */
|
||||
void
|
||||
__gcov_indirect_call_profiler (gcov_type* counter, gcov_type value,
|
||||
void* cur_func, void* callee_func)
|
||||
__gcov_indirect_call_profiler_v2 (gcov_type value, void* cur_func)
|
||||
{
|
||||
/* If the C++ virtual tables contain function descriptors then one
|
||||
function may have multiple descriptors and we need to dereference
|
||||
the descriptors to see if they point to the same function. */
|
||||
if (cur_func == callee_func
|
||||
|| (VTABLE_USES_DESCRIPTORS && callee_func
|
||||
&& *(void **) cur_func == *(void **) callee_func))
|
||||
__gcov_one_value_profiler_body (counter, value);
|
||||
if (cur_func == __gcov_indirect_call_callee
|
||||
|| (VTABLE_USES_DESCRIPTORS && __gcov_indirect_call_callee
|
||||
&& *(void **) cur_func == *(void **) __gcov_indirect_call_callee))
|
||||
__gcov_one_value_profiler_body (__gcov_indirect_call_counters, value);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue