This is the mail archive of the
gcc-patches@gcc.gnu.org
mailing list for the GCC project.
Re: Better memory statistics, take 2
- From: Jan Hubicka <hubicka at ucw dot cz>
- To: Jan Hubicka <hubicka at ucw dot cz>
- Cc: gcc-patches at gcc dot gnu dot org, stevenb at suse dot de, rth at redhat dot com
- Date: Thu, 2 Sep 2004 18:47:01 +0200
- Subject: Re: Better memory statistics, take 2
- References: <20040902161203.GF22834@atrey.karlin.mff.cuni.cz>
> Hi,
> here is updated version of patch I sent while reducing memory for GCC
> 3.4, it is quite usefull now again...
>
> Hi,
> this patch improves the per-line statistics by tracking down each allocated
> entity to figure out whether it will be freed, garbage collected or leaked.
> To rule out ggc_freed values is pretty important as these are much cheaper,
> similarly I used leaks to figure out why C++ frontend use that much of memory.
>
> The tracking is implemented via ptr_hash that converts pointers to records and
> by forcing one garbage collection run before outputting the data so leaks are
> accurate.
> For combine.c we now get:
>
> emit-rtl.c:738 (gen_reg_rtx) 203904: 0.9% 118944:46.1% 0: 0.0% 89376: 7.8% 54
> c-decl.c:4338 (grokdeclarator) 29160: 0.1% 0: 0.0% 316656:10.8% 0: 0.0% 3202
> ggc-common.c:192 (ggc_calloc) 332480: 1.5% 0: 0.0% 51536: 1.7% 4240: 0.4% 575
> optabs.c:4913 (new_convert_optab) 0: 0.0% 0: 0.0% 416916:14.2% 122004:10.7% 9
> tree.c:4010 (build_function_type) 348672: 1.5% 0: 0.0% 135808: 4.6% 75700: 6.6% 3785
> convert.c:371 (convert_to_integer) 517980: 2.3% 0: 0.0% 1480: 0.1% 0: 0.0% 25973
> c-parse.y:748 (yyparse) 521360: 2.3% 0: 0.0% 0: 0.0% 0: 0.0% 26068
> c-parse.y:447 (yyparse) 533540: 2.3% 0: 0.0% 480: 0.0% 0: 0.0% 26701
> tree.c:406 (build_int_2_wide) 529060: 2.3% 0: 0.0% 53020: 1.8% 0: 0.0% 29104
> emit-rtl.c:3367 (make_jump_insn_raw) 643524: 2.8% 0: 0.0% 0: 0.0% 153220:13.4% 7661
> genrtl.c:635 (gen_rtx_fmt_i00) 670160: 2.9% 0: 0.0% 2416: 0.1% 0: 0.0% 42036
> genrtl.c:671 (gen_rtx_fmt_e0) 688752: 3.0% 0: 0.0% 15036: 0.5% 0: 0.0% 58649
> stringpool.c:70 (alloc_node) 0: 0.0% 0: 0.0% 716520:24.3% 170600:15.0% 8530
> c-decl.c:4270 (grokdeclarator) 785160: 3.4% 0: 0.0% 18576: 0.6% 0: 0.0% 7442
> varray.c:128 (varray_init) 793716: 3.5% 15340: 6.0% 10948: 0.4% 211316:18.5% 591
> cselib.c:840 (cselib_subst_to_values) 992512: 4.3% 0: 0.0% 0: 0.0% 0: 0.0% 86828
> emit-rtl.c:3335 (make_insn_raw) 1501800: 6.6% 0: 0.0% 40: 0.0% 0: 0.0% 37546
> genrtl.c:51 (gen_rtx_fmt_ue) 1574316: 6.9% 0: 0.0% 0: 0.0% 0: 0.0% 131193
> rtl.c:250 (copy_rtx) 1723556: 7.5% 0: 0.0% 576: 0.0% 0: 0.0% 141955
> genrtl.c:33 (gen_rtx_fmt_ee) 3201600:14.0% 0: 0.0% 36: 0.0% 0: 0.0% 266803
> Total 22904228 257740 2945464 1139300 1281616
> source location Garbage Freed Leak Overhead Times
>
> So only 257KB is explicitely freed right now. I will try to add few extra calls.
Sorry, I managed to melt two changes together, here is proper patch and
I am just re-testing it.
Honza
Index: ggc-common.c
===================================================================
RCS file: /cvs/gcc/gcc/gcc/ggc-common.c,v
retrieving revision 1.88
diff -c -3 -p -r1.88 ggc-common.c
*** ggc-common.c 9 Aug 2004 20:19:29 -0000 1.88
--- ggc-common.c 2 Sep 2004 16:08:50 -0000
*************** Software Foundation, 59 Temple Place - S
*** 61,66 ****
--- 61,69 ----
#define VALGRIND_DISCARD(x)
#endif
+ /* When set, ggc_collect will do collection. */
+ bool ggc_force_collect;
+
/* Statistics about the allocation. */
static ggc_statistics *ggc_stats;
*************** struct loc_descriptor
*** 780,785 ****
--- 783,790 ----
int times;
size_t allocated;
size_t overhead;
+ size_t freed;
+ size_t collected;
};
/* Hashtable used for statistics. */
*************** eq_descriptor (const void *p1, const voi
*** 804,809 ****
--- 809,840 ----
&& d->function == d2->function);
}
+ /* Hashtable converting address of allocated field to loc descriptor. */
+ static htab_t ptr_hash;
+ struct ptr_hash_entry
+ {
+ void *ptr;
+ struct loc_descriptor *loc;
+ size_t size;
+ };
+
+ /* Hash table helpers functions. */
+ static hashval_t
+ hash_ptr (const void *p)
+ {
+ const struct ptr_hash_entry *d = p;
+
+ return htab_hash_pointer (d->ptr);
+ }
+
+ static int
+ eq_ptr (const void *p1, const void *p2)
+ {
+ const struct ptr_hash_entry *p = p1;
+
+ return (p->ptr == p2);
+ }
+
/* Return descriptor for given call site, create new one if needed. */
static struct loc_descriptor *
loc_descriptor (const char *name, int line, const char *function)
*************** loc_descriptor (const char *name, int li
*** 829,851 ****
/* Record ALLOCATED and OVERHEAD bytes to descriptor NAME:LINE (FUNCTION). */
void
! ggc_record_overhead (size_t allocated, size_t overhead,
const char *name, int line, const char *function)
{
struct loc_descriptor *loc = loc_descriptor (name, line, function);
loc->times++;
loc->allocated+=allocated;
loc->overhead+=overhead;
}
/* Helper for qsort; sort descriptors by amount of memory consumed. */
static int
cmp_statistic (const void *loc1, const void *loc2)
{
struct loc_descriptor *l1 = *(struct loc_descriptor **) loc1;
struct loc_descriptor *l2 = *(struct loc_descriptor **) loc2;
! return (l1->allocated + l1->overhead) - (l2->allocated + l2->overhead);
}
/* Collect array of the descriptors from hashtable. */
--- 860,929 ----
/* Record ALLOCATED and OVERHEAD bytes to descriptor NAME:LINE (FUNCTION). */
void
! ggc_record_overhead (size_t allocated, size_t overhead, void *ptr,
const char *name, int line, const char *function)
{
struct loc_descriptor *loc = loc_descriptor (name, line, function);
+ struct ptr_hash_entry *p = xmalloc (sizeof (struct ptr_hash_entry));
+ PTR *slot;
+
+ p->ptr = ptr;
+ p->loc = loc;
+ p->size = allocated + overhead;
+ if (!ptr_hash)
+ ptr_hash = htab_create (10, hash_ptr, eq_ptr, NULL);
+ slot = htab_find_slot_with_hash (ptr_hash, ptr, htab_hash_pointer (ptr), INSERT);
+ if (*slot)
+ abort ();
+ *slot = p;
loc->times++;
loc->allocated+=allocated;
loc->overhead+=overhead;
}
+ /* Helper function for prune_overhead_list. See if SLOT is still marked and
+ remove it from hashtable if it is not. */
+ static int
+ ggc_prune_ptr (void **slot, void *b ATTRIBUTE_UNUSED)
+ {
+ struct ptr_hash_entry *p = *slot;
+ if (!ggc_marked_p (p->ptr))
+ {
+ p->loc->collected += p->size;
+ htab_clear_slot (ptr_hash, slot);
+ free (p);
+ }
+ return 1;
+ }
+
+ /* After live values has been marked, walk all recorded pointers and see if
+ they are still live. */
+ void
+ ggc_prune_overhead_list (void)
+ {
+ htab_traverse (ptr_hash, ggc_prune_ptr, NULL);
+ }
+
+ /* Notice that the pointer has been freed. */
+ void ggc_free_overhead (void *ptr)
+ {
+ PTR *slot = htab_find_slot_with_hash (ptr_hash, ptr, htab_hash_pointer (ptr),
+ NO_INSERT);
+ struct ptr_hash_entry *p = *slot;
+ p->loc->freed += p->size;
+ htab_clear_slot (ptr_hash, slot);
+ free (p);
+ }
+
/* Helper for qsort; sort descriptors by amount of memory consumed. */
static int
cmp_statistic (const void *loc1, const void *loc2)
{
struct loc_descriptor *l1 = *(struct loc_descriptor **) loc1;
struct loc_descriptor *l2 = *(struct loc_descriptor **) loc2;
! return ((l1->allocated + l1->overhead - l1->freed) -
! (l2->allocated + l2->overhead - l1->freed));
}
/* Collect array of the descriptors from hashtable. */
*************** void dump_ggc_loc_statistics (void)
*** 866,889 ****
#ifdef GATHER_STATISTICS
int nentries = 0;
char s[4096];
! size_t count, size, overhead;
int i;
loc_array = xcalloc (sizeof (*loc_array), loc_hash->n_elements);
fprintf (stderr, "-------------------------------------------------------\n");
! fprintf (stderr, "\n%-60s %10s %10s %10s\n",
! "source location", "Times", "Allocated", "Overhead");
fprintf (stderr, "-------------------------------------------------------\n");
- count = 0;
- size = 0;
- overhead = 0;
htab_traverse (loc_hash, add_statistics, &nentries);
qsort (loc_array, nentries, sizeof (*loc_array), cmp_statistic);
for (i = 0; i < nentries; i++)
{
struct loc_descriptor *d = loc_array[i];
! size += d->allocated;
! count += d->times;
overhead += d->overhead;
}
for (i = 0; i < nentries; i++)
--- 944,969 ----
#ifdef GATHER_STATISTICS
int nentries = 0;
char s[4096];
! size_t collected = 0, freed = 0, allocated = 0, overhead = 0, times = 0;
int i;
+ ggc_force_collect = true;
+ ggc_collect ();
+
loc_array = xcalloc (sizeof (*loc_array), loc_hash->n_elements);
fprintf (stderr, "-------------------------------------------------------\n");
! fprintf (stderr, "\n%-48s %10s %10s %10s %10s %10s\n",
! "source location", "Garbage", "Freed", "Leak", "Overhead", "Times");
fprintf (stderr, "-------------------------------------------------------\n");
htab_traverse (loc_hash, add_statistics, &nentries);
qsort (loc_array, nentries, sizeof (*loc_array), cmp_statistic);
for (i = 0; i < nentries; i++)
{
struct loc_descriptor *d = loc_array[i];
! allocated += d->allocated;
! times += d->times;
! freed += d->freed;
! collected += d->collected;
overhead += d->overhead;
}
for (i = 0; i < nentries; i++)
*************** void dump_ggc_loc_statistics (void)
*** 896,908 ****
while ((s2 = strstr (s1, "gcc/")))
s1 = s2 + 4;
sprintf (s, "%s:%i (%s)", s1, d->line, d->function);
! fprintf (stderr, "%-60s %10i %10li %10li:%.3f%%\n", s,
! d->times, (long)d->allocated, (long)d->overhead,
! (d->allocated + d->overhead) *100.0 / (size + overhead));
}
}
! fprintf (stderr, "%-60s %10ld %10ld %10ld\n",
! "Total", (long)count, (long)size, (long)overhead);
fprintf (stderr, "-------------------------------------------------------\n");
#endif
}
--- 976,1001 ----
while ((s2 = strstr (s1, "gcc/")))
s1 = s2 + 4;
sprintf (s, "%s:%i (%s)", s1, d->line, d->function);
! s[48] = 0;
! fprintf (stderr, "%-48s %10li:%4.1f%% %10li:%4.1f%% %10li:%4.1f%% %10li:%4.1f%% %10li\n", s,
! (long)d->collected,
! (d->collected) * 100.0 / collected,
! (long)d->freed,
! (d->freed) * 100.0 / freed,
! (long)(d->allocated + d->overhead - d->freed - d->collected),
! (d->allocated + d->overhead - d->freed - d->collected) * 100.0
! / (allocated + overhead - freed - collected),
! (long)d->overhead,
! d->overhead * 100.0 / overhead,
! (long)d->times);
}
}
! fprintf (stderr, "%-48s %10ld %10ld %10ld %10ld %10ld\n",
! "Total", (long)collected, (long)freed,
! (long)(allocated + overhead - freed - collected), (long)overhead,
! (long)times);
! fprintf (stderr, "%-48s %10s %10s %10s %10s %10s\n",
! "source location", "Garbage", "Freed", "Leak", "Overhead", "Times");
fprintf (stderr, "-------------------------------------------------------\n");
#endif
}
Index: ggc-page.c
===================================================================
RCS file: /cvs/gcc/gcc/gcc/ggc-page.c,v
retrieving revision 1.92
diff -c -3 -p -r1.92 ggc-page.c
*** ggc-page.c 22 Mar 2004 02:57:27 -0000 1.92
--- ggc-page.c 2 Sep 2004 16:08:50 -0000
*************** ggc_alloc_stat (size_t size MEM_STAT_DEC
*** 1173,1184 ****
G.page_tails[order]->next = entry;
G.page_tails[order] = entry;
}
- #ifdef GATHER_STATISTICS
- ggc_record_overhead (OBJECT_SIZE (order), OBJECT_SIZE (order) - size PASS_MEM_STAT);
- #endif
/* Calculate the object's address. */
result = entry->page + object_offset;
#ifdef ENABLE_GC_CHECKING
/* Keep poisoning-by-writing-0xaf the object, in an attempt to keep the
--- 1173,1185 ----
G.page_tails[order]->next = entry;
G.page_tails[order] = entry;
}
/* Calculate the object's address. */
result = entry->page + object_offset;
+ #ifdef GATHER_STATISTICS
+ ggc_record_overhead (OBJECT_SIZE (order), OBJECT_SIZE (order) - size,
+ result PASS_MEM_STAT);
+ #endif
#ifdef ENABLE_GC_CHECKING
/* Keep poisoning-by-writing-0xaf the object, in an attempt to keep the
*************** ggc_free (void *p)
*** 1327,1332 ****
--- 1328,1337 ----
size_t order = pe->order;
size_t size = OBJECT_SIZE (order);
+ #ifdef GATHER_STATISTICS
+ ggc_free_overhead (p);
+ #endif
+
if (GGC_DEBUG_LEVEL >= 3)
fprintf (G.debug_file,
"Freeing object, actual size=%lu, at %p on %p\n",
*************** ggc_collect (void)
*** 1971,1977 ****
float min_expand = allocated_last_gc * PARAM_VALUE (GGC_MIN_EXPAND) / 100;
! if (G.allocated < allocated_last_gc + min_expand)
return;
timevar_push (TV_GC);
--- 1976,1982 ----
float min_expand = allocated_last_gc * PARAM_VALUE (GGC_MIN_EXPAND) / 100;
! if (G.allocated < allocated_last_gc + min_expand && !ggc_force_collect)
return;
timevar_push (TV_GC);
*************** ggc_collect (void)
*** 1993,1998 ****
--- 1998,2006 ----
clear_marks ();
ggc_mark_roots ();
+ #ifdef GATHER_STATISTICS
+ ggc_prune_overhead_list ();
+ #endif
poison_pages ();
validate_free_objects ();
sweep_pages ();
Index: ggc.h
===================================================================
RCS file: /cvs/gcc/gcc/gcc/ggc.h,v
retrieving revision 1.68
diff -c -3 -p -r1.68 ggc.h
*** ggc.h 2 Sep 2004 02:39:15 -0000 1.68
--- ggc.h 2 Sep 2004 16:08:50 -0000
*************** extern struct alloc_zone *garbage_zone;
*** 209,214 ****
--- 209,216 ----
extern struct alloc_zone *rtl_zone;
/* For regular tree allocations. */
extern struct alloc_zone *tree_zone;
+ /* When set, ggc_collect will do collection. */
+ extern bool ggc_force_collect;
/* The internal primitive. */
extern void *ggc_alloc_stat (size_t MEM_STAT_DECL);
*************** extern void *ggc_calloc (size_t, size_t)
*** 233,239 ****
/* Free a block. To be used when known for certain it's not reachable. */
extern void ggc_free (void *);
! extern void ggc_record_overhead (size_t, size_t MEM_STAT_DECL);
extern void dump_ggc_loc_statistics (void);
--- 235,243 ----
/* Free a block. To be used when known for certain it's not reachable. */
extern void ggc_free (void *);
! extern void ggc_record_overhead (size_t, size_t, void * MEM_STAT_DECL);
! extern void ggc_free_overhead (void *);
! extern void ggc_prune_overhead_list (void);
extern void dump_ggc_loc_statistics (void);