This is the mail archive of the
gcc@gcc.gnu.org
mailing list for the GCC project.
Re: Let's backport speedups from 3.4 to 3.3
- From: Kaoru Fukui <k_fukui at highway dot ne dot jp>
- To: ghazi at caip dot rutgers dot edu
- Cc: gcc at gcc dot gnu dot org, gcc-patches at gcc dot gnu dot org
- Date: Thu, 13 Feb 2003 02:50:14 +0900 (JST)
- Subject: Re: Let's backport speedups from 3.4 to 3.3
- Reply-to: Kaoru Fukui <k_fukui at highway dot ne dot jp>
On 13 Feb, To: ghazi@caip.rutgers.edu wrote:
> On 12 Feb, Kaveh R. Ghazi wrote:
>> I've heard some mention that 3.4 is faster in some cases than 3.3.
>>
>> I know several speedup tweeks have been made since the branch was
>> created. Not all of them have been backported.
>>
>> If they're not too invasive, let's consider them for 3.3. I'm willing
>> to help by regtesting patches if people can point out safe candidates.
>> Though I'd certainly appreciate assistance from others. :-)
>>
>> Thanks,
>> --Kaveh
>> --
>> Kaveh R. Ghazi ghazi@caip.rutgers.edu
>
> http://gcc.gnu.org/ml/gcc-patches/2003-02/msg00377.html
> http://gcc.gnu.org/ml/gcc-patches/2003-02/msg00560.htm
> There is Mike's info.
>
>
> I tried popcontext-fsf-2.diffs which is speedy, see below
>
> http://gcc.gnu.org/ml/gcc-patches/2003-02/msg00380.html
> gcc-3.3 doesn't pch, so it needs to adjust.
This is my adjusted Mike's patch for gcc-3.3.
Thanks Mike
Kaoru
====================================
* ggc-page.c (struct page_entry): Add index_by_depth field.
Remove save_in_use_p field.
(struct globals): Add by_depth, depth, and save_in_use fields.
(INITIAL_PTE_COUNT): Add.
(save_in_use_p_i): Add.
(save_in_use_p): Add.
(adjust_depth): Add.
(move_ptes_to_front): Add.
(free_page): Add support for and use faster data structures.
(ggc_alloc): Likewise.
(init_ggc): Likewise.
(ggc_recalculate_in_use_p): Likewise.
(ggc_pop_context): Likewise.
(clear_marks): Likewise.
(ggc_pch_read): Likewise.
(ggc_varray_type): Add.
(ggc_varray_head_tag): Add.
(ggc_varray_data_enum): Add.
(GGC_VARRAY_DATA_C): Add.
(GGC_VARRAY_DATA_GENERICNGC): Add.
(GGC_VARRAY_DATA_UNGC): Add.
(GGC_NUM_VARRAY_DATA): Add.
(union ggc_varray_data_tag): Add.
(ggc_varray_data): Add.
(ggc_varray_init): Add.
(GGC_VARRAY_GENERIC_PTRNGC_INIT): Add.
(GGC_VARRAY_UINTNGC_INIT): Add.
(GGC_VARRAY_GENERIC_PTRNGC): Add.
(GGC_VARRAY_UINTNGC): Add.
(GGC_VARRAY_PUSH_GENERIC_PTRNGC): Add.
(GGC_VARRAY_PUSH_UINTNGC): Add.
(GGC_VARRAY_TOP_GENERIC_PTRNGC): Add.
(GGC_VARRAY_TOP_UINTNGC): Add.
(GGC_VARRAY_FREE): Add.
(GGC_VARRAY_GROW): Add.
(GGC_VARRAY_PUSH): Add.
(GGC_VARRAY_POP): Add.
(GGC_VARRAY_TOP): Add.
(GGC_VARRAY_HDR_SIZE): Add.
(ggc_varray_init): Add.
(ggc_varray_grow): Add.
(ggc_varray_check_failed): Add.
(GGC_VARRAY_ACTIVE_SIZE): Add.
* varray.c (element_size): Remove.
(uses_ggc): Remove.
(element): Add.
(varray_init): Use new interface.
(varray_grow): Use new interface.
(varray_clear): Use new interface.
Index: varray.c
===================================================================
RCS file: /cvs/gcc/gcc/gcc/varray.c,v
retrieving revision 1.16
diff -p -r1.16 varray.c
*** varray.c 16 Dec 2002 18:20:02 -0000 1.16
--- varray.c 6 Feb 2003 01:54:07 -0000
***************
*** 29,62 ****
#define VARRAY_HDR_SIZE (sizeof (struct varray_head_tag) - sizeof (varray_data))
! static const size_t element_size[NUM_VARRAY_DATA] = {
! sizeof (char),
! sizeof (unsigned char),
! sizeof (short),
! sizeof (unsigned short),
! sizeof (int),
! sizeof (unsigned int),
! sizeof (long),
! sizeof (unsigned long),
! sizeof (HOST_WIDE_INT),
! sizeof (unsigned HOST_WIDE_INT),
! sizeof (PTR),
! sizeof (char *),
! sizeof (struct rtx_def *),
! sizeof (struct rtvec_def *),
! sizeof (union tree_node *),
! sizeof (struct bitmap_head_def *),
! sizeof (struct reg_info_def *),
! sizeof (struct const_equiv_data),
! sizeof (struct basic_block_def *),
! sizeof (struct elt_list *)
! };
! static const int uses_ggc[NUM_VARRAY_DATA] = {
! 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* unsigned HOST_WIDE_INT */
! 1, /* PTR */
! 1, 1, 1, 1, 1, /* bitmap_head_def */
! 0, 0, 0, 1
};
/* Allocate a virtual array with NUM_ELEMENT elements, each of which is
--- 29,61 ----
#define VARRAY_HDR_SIZE (sizeof (struct varray_head_tag) - sizeof (varray_data))
! /* Do not add any more non-GC items here. Please either remove or GC those items that
! are not GCed. */
! static const struct {
! size_t size;
! int uses_ggc;
! } element[NUM_VARRAY_DATA] = {
! { sizeof (char), 1 },
! { sizeof (unsigned char), 1 },
! { sizeof (short), 1 },
! { sizeof (unsigned short), 1 },
! { sizeof (int), 1 },
! { sizeof (unsigned int), 1 },
! { sizeof (long), 1 },
! { sizeof (unsigned long), 1 },
! { sizeof (HOST_WIDE_INT), 1 },
! { sizeof (unsigned HOST_WIDE_INT), 1 },
! { sizeof (PTR), 1 },
! { sizeof (char *), 1 },
! { sizeof (struct rtx_def *), 1 },
! { sizeof (struct rtvec_def *), 1 },
! { sizeof (union tree_node *), 1 },
! { sizeof (struct bitmap_head_def *), 1 },
! { sizeof (struct reg_info_def *), 0 },
! { sizeof (struct const_equiv_data), 0 },
! { sizeof (struct basic_block_def *), 0 },
! { sizeof (struct elt_list *), 1 },
};
/* Allocate a virtual array with NUM_ELEMENT elements, each of which is
*************** varray_init (num_elements, element_kind,
*** 67,75 ****
enum varray_data_enum element_kind;
const char *name;
{
! size_t data_size = num_elements * element_size[element_kind];
varray_type ptr;
! if (uses_ggc [element_kind])
ptr = (varray_type) ggc_alloc_cleared (VARRAY_HDR_SIZE + data_size);
else
ptr = (varray_type) xcalloc (VARRAY_HDR_SIZE + data_size, 1);
--- 66,74 ----
enum varray_data_enum element_kind;
const char *name;
{
! size_t data_size = num_elements * element[element_kind].size;
varray_type ptr;
! if (element[element_kind].uses_ggc)
ptr = (varray_type) ggc_alloc_cleared (VARRAY_HDR_SIZE + data_size);
else
ptr = (varray_type) xcalloc (VARRAY_HDR_SIZE + data_size, 1);
*************** varray_grow (va, n)
*** 92,102 ****
if (n != old_elements)
{
! size_t elem_size = element_size[va->type];
size_t old_data_size = old_elements * elem_size;
size_t data_size = n * elem_size;
! if (uses_ggc[va->type])
va = (varray_type) ggc_realloc (va, VARRAY_HDR_SIZE + data_size);
else
va = (varray_type) xrealloc ((char *) va, VARRAY_HDR_SIZE + data_size);
--- 91,101 ----
if (n != old_elements)
{
! size_t elem_size = element[va->type].size;
size_t old_data_size = old_elements * elem_size;
size_t data_size = n * elem_size;
! if (element[va->type].uses_ggc)
va = (varray_type) ggc_realloc (va, VARRAY_HDR_SIZE + data_size);
else
va = (varray_type) xrealloc ((char *) va, VARRAY_HDR_SIZE + data_size);
*************** void
*** 113,119 ****
varray_clear (va)
varray_type va;
{
! size_t data_size = element_size[va->type] * va->num_elements;
memset (va->data.c, 0, data_size);
va->elements_used = 0;
--- 112,118 ----
varray_clear (va)
varray_type va;
{
! size_t data_size = element[va->type].size * va->num_elements;
memset (va->data.c, 0, data_size);
va->elements_used = 0;
Index: varray.h
===================================================================
RCS file: /cvs/gcc/gcc/gcc/varray.h,v
retrieving revision 1.29
diff -p -r1.29 varray.h
*** varray.h 16 Dec 2002 18:20:02 -0000 1.29
--- varray.h 6 Feb 2003 01:54:07 -0000
*************** struct const_equiv_data GTY(()) {
*** 57,63 ****
};
/* Enum indicating what the varray contains.
! If this is changed, `element_size' in varray.c needs to be updated. */
enum varray_data_enum {
VARRAY_DATA_C,
--- 57,63 ----
};
/* Enum indicating what the varray contains.
! If this is changed, `element' in varray.c needs to be updated. */
enum varray_data_enum {
VARRAY_DATA_C,
Index: ggc-page.c
===================================================================
RCS file: /cvs/gcc/gcc/gcc/ggc-page.c,v
retrieving revision 1.61
diff -p -r1.61 ggc-page.c
*** ggc-page.c 30 Jan 2003 18:14:06 -0000 1.61
--- ggc-page.c 6 Feb 2003 01:54:09 -0000
*************** Software Foundation, 59 Temple Place - S
*** 26,32 ****
#include "rtl.h"
#include "tm_p.h"
#include "toplev.h"
- #include "varray.h"
#include "flags.h"
#include "ggc.h"
#include "timevar.h"
--- 26,31 ----
*************** typedef struct page_entry
*** 253,261 ****
struct page_group *group;
#endif
! /* Saved in-use bit vector for pages that aren't in the topmost
! context during collection. */
! unsigned long *save_in_use_p;
/* Context depth of this page. */
unsigned short context_depth;
--- 252,260 ----
struct page_group *group;
#endif
! /* This is the index in the by_depth varray where this page table
! can be found. */
! unsigned long index_by_depth;
/* Context depth of this page. */
unsigned short context_depth;
*************** typedef struct page_table_chain
*** 314,319 ****
--- 313,320 ----
#endif
+ typedef struct ggc_varray_head_tag *ggc_varray_type;
+
/* The rest of the global variables. */
static struct globals
{
*************** static struct globals
*** 367,372 ****
--- 368,391 ----
/* The file descriptor for debugging output. */
FILE *debug_file;
+
+ /* Each element of this array is a page_entry, all page_entries can
+ be found in here by increasing depth. index_by_depth in the
+ page_entry is the index into this data structure where that
+ page_entry can be found. This is used to speed up finding all
+ page_entries at a particular depth. */
+ ggc_varray_type by_depth;
+
+ /* Each element is an index in by_depth where the given depth
+ starts. This structure is indexed by that given depth we are
+ interested in. */
+ ggc_varray_type depth;
+
+ /* Each element is a pointer to the saved in_use_p bits, if any,
+ zero otherwise. We allocate them all together, to enable a
+ better runtime data access pattern. */
+ ggc_varray_type save_in_use;
+
} G;
/* The size in bytes required to maintain a bitmap for the objects
*************** static struct globals
*** 379,384 ****
--- 398,406 ----
free list. This cannot be larger than HOST_BITS_PER_INT for the
in_use bitmask for page_group. */
#define GGC_QUIRE_SIZE 16
+
+ /* Initial guess as to how many page table entries we might need. */
+ #define INITIAL_PTE_COUNT 128
static int ggc_allocated_p PARAMS ((const void *));
static page_entry *lookup_page_table_entry PARAMS ((const void *));
*************** static void clear_marks PARAMS ((void));
*** 398,403 ****
--- 420,427 ----
static void sweep_pages PARAMS ((void));
static void ggc_recalculate_in_use_p PARAMS ((page_entry *));
static void compute_inverse PARAMS ((unsigned));
+ static inline void adjust_depth PARAMS ((void));
+ static void move_ptes_to_front PARAMS ((int, int));
#ifdef ENABLE_GC_CHECKING
static void poison_pages PARAMS ((void));
*************** static void poison_pages PARAMS ((void))
*** 405,410 ****
--- 429,597 ----
void debug_print_page_list PARAMS ((int));
+ /* Internal varray management code. This is separated from varrays to
+ enable the removal of non-GCed objects from varray.[ch] */
+
+ enum ggc_varray_data_enum {
+ GGC_VARRAY_DATA_C,
+ GGC_VARRAY_DATA_GENERICNGC,
+ GGC_VARRAY_DATA_UNGC,
+ GGC_NUM_VARRAY_DATA
+ };
+
+ /* Union of various array types that are used. */
+ typedef union ggc_varray_data_tag {
+ char c[1];
+ PTR genericngc[1];
+ unsigned int ungc[1];
+ } ggc_varray_data;
+
+ /* Virtual array of pointers header. */
+ struct ggc_varray_head_tag {
+ size_t num_elements; /* Maximum element number allocated. */
+ size_t elements_used; /* The number of elements used, if
+ using GGC_VARRAY_PUSH/GGC_VARRAY_POP. */
+ enum ggc_varray_data_enum type; /* The kind of elements in the varray. */
+ const char *name; /* name of the varray for reporting errors */
+ ggc_varray_data data; /* The data elements follow, must be last. */
+ };
+
+ /* Allocate a virtual array with NUM elements, each of which is SIZE bytes
+ long, named NAME. Array elements are zeroed. */
+ static ggc_varray_type ggc_varray_init PARAMS ((size_t, enum ggc_varray_data_enum,
+ const char *));
+
+ #define GGC_VARRAY_GENERIC_PTRNGC_INIT(va, num, name) \
+ va = ggc_varray_init (num, GGC_VARRAY_DATA_GENERICNGC, name)
+
+ #define GGC_VARRAY_UINTNGC_INIT(va, num, name) \
+ va = ggc_varray_init (num, GGC_VARRAY_DATA_UNGC, name)
+
+ #define GGC_VARRAY_GENERIC_PTRNGC(VA,N) GGC_VARRAY_CHECK (VA, N, genericngc)
+ #define GGC_VARRAY_UINTNGC(VA, N) GGC_VARRAY_CHECK (VA, N, ungc)
+
+ /* Free up memory allocated by the virtual array, but do not free any of the
+ elements involved. */
+ #define GGC_VARRAY_FREE(vp) \
+ do { if (vp) { free (vp); vp = (ggc_varray_type) 0; } } while (0)
+ #define GGC_VARRAY_GROW(VA, N) ((VA) = ggc_varray_grow (VA, N))
+
+ /* Push X onto VA. T is the name of the field in varray_data
+ corresponding to the type of X. */
+ #define GGC_VARRAY_PUSH(VA, T, X) \
+ do \
+ { \
+ if ((VA)->elements_used >= (VA)->num_elements) \
+ GGC_VARRAY_GROW ((VA), 2 * (VA)->num_elements); \
+ (VA)->data.T[(VA)->elements_used++] = (X); \
+ } \
+ while (0)
+
+ #define GGC_VARRAY_PUSH_GENERIC_PTRNGC(VA, X) GGC_VARRAY_PUSH (VA, genericngc, X)
+ #define GGC_VARRAY_PUSH_UINTNGC(VA, X) GGC_VARRAY_PUSH (VA, ungc, X)
+
+ /* Pop the top element of VA. */
+ #define GGC_VARRAY_POP(VA) \
+ ((VA)->elements_used--)
+
+ /* Return the top element of VA. */
+ #define GGC_VARRAY_TOP(VA, T) \
+ ((VA)->data.T[(VA)->elements_used - 1])
+ #define GGC_VARRAY_TOP_GENERIC_PTRNGC(VA) GGC_VARRAY_TOP (VA, genericngc)
+ #define GGC_VARRAY_TOP_UINTNGC(VA) GGC_VARRAY_TOP (VA, ungc)
+
+ #define GGC_VARRAY_HDR_SIZE (sizeof (struct ggc_varray_head_tag) - sizeof (ggc_varray_data))
+
+ static const size_t element[GGC_NUM_VARRAY_DATA] = {
+ sizeof (char),
+ sizeof (PTR),
+ sizeof (unsigned int)
+ };
+
+ static inline ggc_varray_type ggc_varray_grow PARAMS ((ggc_varray_type, size_t));
+
+ /* Allocate a virtual array with NUM_ELEMENT elements, each of which is
+ ELEMENT_SIZE bytes long, named NAME. Array elements are zeroed. */
+ static ggc_varray_type
+ ggc_varray_init (num_elements, element_kind, name)
+ size_t num_elements;
+ enum ggc_varray_data_enum element_kind;
+ const char *name;
+ {
+ size_t data_size = num_elements * element[element_kind];
+ ggc_varray_type ptr;
+ ptr = (ggc_varray_type) xcalloc (GGC_VARRAY_HDR_SIZE + data_size, 1);
+
+ ptr->num_elements = num_elements;
+ ptr->elements_used = 0;
+ ptr->type = element_kind;
+ ptr->name = name;
+ return ptr;
+ }
+
+ /* Grow/shrink the virtual array VA to N elements. Zero any new elements
+ allocated. */
+ static inline ggc_varray_type
+ ggc_varray_grow (va, n)
+ ggc_varray_type va;
+ size_t n;
+ {
+ size_t old_elements = va->num_elements;
+
+ if (n != old_elements)
+ {
+ size_t elem_size = element[va->type];
+ size_t old_data_size = old_elements * elem_size;
+ size_t data_size = n * elem_size;
+
+ va = (ggc_varray_type) xrealloc ((char *) va, GGC_VARRAY_HDR_SIZE + data_size);
+ va->num_elements = n;
+ if (n > old_elements)
+ memset (&va->data.c[old_data_size], 0, data_size - old_data_size);
+ }
+
+ return va;
+ }
+
+ /* Check the bounds of a varray access. */
+
+ #if defined ENABLE_CHECKING && (GCC_VERSION >= 2007)
+ extern void ggc_varray_check_failed PARAMS ((ggc_varray_type, size_t,
+ const char *, int,
+ const char *)) ATTRIBUTE_NORETURN;
+ #define GGC_VARRAY_CHECK(VA, N, T) __extension__ \
+ (*({ ggc_varray_type const _va = (VA); \
+ const size_t _n = (N); \
+ if (_n >= _va->num_elements) \
+ ggc_varray_check_failed (_va, _n, __FILE__, __LINE__, __FUNCTION__); \
+ &_va->data.T[_n]; }))
+
+ extern void error PARAMS ((const char *, ...)) ATTRIBUTE_PRINTF_1;
+
+ void
+ ggc_varray_check_failed (va, n, file, line, function)
+ ggc_varray_type va;
+ size_t n;
+ const char *file;
+ int line;
+ const char *function;
+ {
+ internal_error ("virtual array %s[%lu]: element %lu out of bounds in %s, at %s:%d",
+ va->name, (unsigned long) va->num_elements, (unsigned long) n,
+ function, trim_filename (file), line);
+ }
+ #else
+ #define GGC_VARRAY_CHECK(VA, N, T) ((VA)->data.T[N])
+ #endif
+
+ #define GGC_VARRAY_ACTIVE_SIZE(VA) ((VA)->elements_used)
+
+
+ #define save_in_use_p_i(__i) \
+ (*(unsigned long **)&GGC_VARRAY_GENERIC_PTRNGC (G.save_in_use, __i))
+ #define save_in_use_p(__p) \
+ (save_in_use_p_i (__p->index_by_depth))
+
/* Returns nonzero if P was allocated in GC'able memory. */
static inline int
*************** alloc_page (order)
*** 772,777 ****
--- 959,983 ----
return entry;
}
+ /* Adjust the size of G.depth so that no index greater than the one
+ used by the top of the G.by_depth varray is used. */
+
+ static inline void
+ adjust_depth ()
+ {
+ page_entry *top;
+
+ if (GGC_VARRAY_ACTIVE_SIZE (G.by_depth))
+ {
+ top = GGC_VARRAY_TOP_GENERIC_PTRNGC (G.by_depth);
+ /* Peel back indicies in depth that index into by_depth, so that
+ as new elements are added to by_depth, we note the indicies
+ of those elements, if they are for new context depths. */
+ while (GGC_VARRAY_ACTIVE_SIZE (G.depth) > (size_t)top->context_depth+1)
+ GGC_VARRAY_POP (G.depth);
+ }
+ }
+
/* For a page that is no longer needed, put it on the free page list. */
static inline void
*************** free_page (entry)
*** 793,798 ****
--- 999,1030 ----
clear_page_group_in_use (entry->group, entry->page);
#endif
+ if (GGC_VARRAY_ACTIVE_SIZE (G.by_depth) > 1)
+ {
+ page_entry *top = GGC_VARRAY_TOP_GENERIC_PTRNGC (G.by_depth);
+
+ /* If they are at the same depth, put top element into freed
+ slot. */
+ if (entry->context_depth == top->context_depth)
+ {
+ int i = entry->index_by_depth;
+ GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, i) = top;
+ GGC_VARRAY_GENERIC_PTRNGC (G.save_in_use, i)
+ = GGC_VARRAY_TOP_GENERIC_PTRNGC (G.save_in_use);
+ top->index_by_depth = i;
+ }
+ else
+ {
+ /* We cannot free a page from a context deeper than the
+ current one. */
+ abort ();
+ }
+ }
+ GGC_VARRAY_POP (G.by_depth);
+ GGC_VARRAY_POP (G.save_in_use);
+
+ adjust_depth ();
+
entry->next = G.free_pages;
G.free_pages = entry;
}
*************** ggc_alloc (size)
*** 916,921 ****
--- 1148,1162 ----
struct page_entry *new_entry;
new_entry = alloc_page (order);
+ new_entry->index_by_depth = GGC_VARRAY_ACTIVE_SIZE (G.by_depth);
+ GGC_VARRAY_PUSH_GENERIC_PTRNGC (G.by_depth, new_entry);
+ GGC_VARRAY_PUSH_GENERIC_PTRNGC (G.save_in_use, 0);
+
+ /* We can skip context depths, if we do, make sure we go all the
+ way to the new depth. */
+ while (new_entry->context_depth >= GGC_VARRAY_ACTIVE_SIZE (G.depth))
+ GGC_VARRAY_PUSH_UINTNGC (G.depth, GGC_VARRAY_ACTIVE_SIZE (G.by_depth)-1);
+
/* If this is the only entry, it's also the tail. */
if (entry == NULL)
G.page_tails[order] = new_entry;
*************** init_ggc ()
*** 1218,1223 ****
--- 1459,1468 ----
for (i = OBJECT_SIZE (order); size_lookup [i] == o; --i)
size_lookup[i] = order;
}
+
+ GGC_VARRAY_GENERIC_PTRNGC_INIT (G.by_depth, INITIAL_PTE_COUNT, "by_depth");
+ GGC_VARRAY_GENERIC_PTRNGC_INIT (G.save_in_use, INITIAL_PTE_COUNT, "save_in_use");
+ GGC_VARRAY_UINTNGC_INIT (G.depth, 10, "depth");
}
/* Increment the `GC context'. Objects allocated in an outer context
*************** ggc_recalculate_in_use_p (p)
*** 1260,1266 ****
/* Something is in use if it is marked, or if it was in use in a
context further down the context stack. */
! p->in_use_p[i] |= p->save_in_use_p[i];
/* Decrement the free object count for every object allocated. */
for (j = p->in_use_p[i]; j; j >>= 1)
--- 1505,1511 ----
/* Something is in use if it is marked, or if it was in use in a
context further down the context stack. */
! p->in_use_p[i] |= save_in_use_p (p)[i];
/* Decrement the free object count for every object allocated. */
for (j = p->in_use_p[i]; j; j >>= 1)
*************** void
*** 1278,1284 ****
ggc_pop_context ()
{
unsigned long omask;
! unsigned order, depth;
depth = --G.context_depth;
omask = (unsigned long)1 << (depth + 1);
--- 1523,1532 ----
ggc_pop_context ()
{
unsigned long omask;
! unsigned depth, i, e;
! #ifdef ENABLE_CHECKING
! unsigned order;
! #endif
depth = --G.context_depth;
omask = (unsigned long)1 << (depth + 1);
*************** ggc_pop_context ()
*** 1290,1298 ****
G.context_depth_allocations &= omask - 1;
G.context_depth_collections &= omask - 1;
! /* Any remaining pages in the popped context are lowered to the new
! current context; i.e. objects allocated in the popped context and
! left over are imported into the previous context. */
for (order = 2; order < NUM_ORDERS; order++)
{
page_entry *p;
--- 1538,1603 ----
G.context_depth_allocations &= omask - 1;
G.context_depth_collections &= omask - 1;
! /* The G.depth array is shortend so that the last index is the
! context_depth of the top element of by_depth. */
! if (depth+1 < GGC_VARRAY_ACTIVE_SIZE (G.depth))
! e = GGC_VARRAY_UINTNGC (G.depth, depth+1);
! else
! e = GGC_VARRAY_ACTIVE_SIZE (G.by_depth);
!
! /* We might not have any PTEs of depth depth. */
! if (depth < GGC_VARRAY_ACTIVE_SIZE (G.depth))
! {
!
! /* First we go through all the pages at depth depth to
! recalculate the in use bits. */
! for (i = GGC_VARRAY_UINTNGC (G.depth, depth); i < e; ++i)
! {
! page_entry *p;
!
! #ifdef ENABLE_CHECKING
! p = GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, i);
!
! /* Check that all of the pages really are at the depth that
! we expect. */
! if (p->context_depth != depth)
! abort ();
! if (p->index_by_depth != i)
! abort ();
! #endif
!
! __builtin_prefetch (&save_in_use_p_i (i+8));
! __builtin_prefetch (&save_in_use_p_i (i+16));
! if (save_in_use_p_i (i))
! {
! p = GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, i);
! ggc_recalculate_in_use_p (p);
! free (save_in_use_p_i (i));
! save_in_use_p_i (i) = 0;
! }
! }
! }
!
! /* Then, we reset all page_entries with a depth greater than depth
! to be at depth. */
! for (i = e; i < GGC_VARRAY_ACTIVE_SIZE (G.by_depth); ++i)
! {
! page_entry *p = GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, i);
!
! /* Check that all of the pages really are at the depth we
! expect. */
! #ifdef ENABLE_CHECKING
! if (p->context_depth <= depth)
! abort ();
! if (p->index_by_depth != i)
! abort ();
! #endif
! p->context_depth = depth;
! }
!
! adjust_depth ();
!
! #ifdef ENABLE_CHECKING
for (order = 2; order < NUM_ORDERS; order++)
{
page_entry *p;
*************** ggc_pop_context ()
*** 1300,1317 ****
for (p = G.pages[order]; p != NULL; p = p->next)
{
if (p->context_depth > depth)
! p->context_depth = depth;
!
! /* If this page is now in the topmost context, and we'd
! saved its allocation state, restore it. */
! else if (p->context_depth == depth && p->save_in_use_p)
! {
! ggc_recalculate_in_use_p (p);
! free (p->save_in_use_p);
! p->save_in_use_p = 0;
! }
}
}
}
/* Unmark all objects. */
--- 1605,1616 ----
for (p = G.pages[order]; p != NULL; p = p->next)
{
if (p->context_depth > depth)
! abort ();
! else if (p->context_depth == depth && save_in_use_p (p))
! abort ();
}
}
+ #endif
}
/* Unmark all objects. */
*************** clear_marks ()
*** 1341,1349 ****
marks. So, back them up first. */
if (p->context_depth < G.context_depth)
{
! if (! p->save_in_use_p)
! p->save_in_use_p = xmalloc (bitmap_size);
! memcpy (p->save_in_use_p, p->in_use_p, bitmap_size);
}
/* Reset reset the number of free objects and clear the
--- 1640,1648 ----
marks. So, back them up first. */
if (p->context_depth < G.context_depth)
{
! if (! save_in_use_p (p))
! save_in_use_p (p) = xmalloc (bitmap_size);
! memcpy (save_in_use_p (p), p->in_use_p, bitmap_size);
}
/* Reset reset the number of free objects and clear the
*************** ggc_pch_finish (d, f)
*** 1777,1782 ****
--- 2076,2148 ----
free (d);
}
+ /* Move the PCH PTE entries just added to the end of by_depth, to the
+ front. */
+
+ static void
+ move_ptes_to_front (count_old_page_tables, count_new_page_tables)
+ int count_old_page_tables;
+ int count_new_page_tables;
+ {
+ unsigned i;
+
+ /* First, we swap the new entries to the front of the varrays. */
+
+ ggc_varray_type new_by_depth;
+ ggc_varray_type new_save_in_use;
+ int c;
+
+ c = count_old_page_tables + count_new_page_tables;
+ /* Allow for at least 25% growth without resizing. */
+ c = c + c/4;
+ c = MAX (c, INITIAL_PTE_COUNT);
+
+ GGC_VARRAY_GENERIC_PTRNGC_INIT (new_by_depth,
+ c,
+ "by_depth");
+ GGC_VARRAY_GENERIC_PTRNGC_INIT (new_save_in_use,
+ c,
+ "save_in_use");
+ GGC_VARRAY_ACTIVE_SIZE (new_by_depth) = count_old_page_tables
+ + count_new_page_tables;
+ GGC_VARRAY_ACTIVE_SIZE (new_save_in_use) = count_old_page_tables
+ + count_new_page_tables;
+
+ memcpy (&GGC_VARRAY_GENERIC_PTRNGC (new_by_depth, 0),
+ &GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, count_old_page_tables),
+ count_new_page_tables * sizeof (void *));
+ memcpy (&GGC_VARRAY_GENERIC_PTRNGC (new_by_depth, count_new_page_tables),
+ &GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, 0),
+ count_old_page_tables * sizeof (void *));
+ memcpy (&GGC_VARRAY_GENERIC_PTRNGC (new_save_in_use, 0),
+ &GGC_VARRAY_GENERIC_PTRNGC (G.save_in_use, count_old_page_tables),
+ count_new_page_tables * sizeof (void *));
+ memcpy (&GGC_VARRAY_GENERIC_PTRNGC (new_save_in_use, count_new_page_tables),
+ &GGC_VARRAY_GENERIC_PTRNGC (G.save_in_use, 0),
+ count_old_page_tables * sizeof (void *));
+
+ GGC_VARRAY_FREE (G.by_depth);
+ GGC_VARRAY_FREE (G.save_in_use);
+
+ G.by_depth = new_by_depth;
+ G.save_in_use = new_save_in_use;
+
+ /* Now update all the index_by_depth fields. */
+ for (i = GGC_VARRAY_ACTIVE_SIZE (G.by_depth); i > 0; --i)
+ {
+ page_entry *p = GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, i-1);
+ p->index_by_depth = i-1;
+ }
+
+ /* And last, we update the depth pointers in G.depth. The first
+ entry is already 0, and context 0 entries always start at index
+ 0, so there is nothing to update in the first slot. We need a
+ second slot, only if we have old ptes, and if we do, they start
+ at index count_new_page_tables. */
+ if (count_old_page_tables)
+ GGC_VARRAY_PUSH_UINTNGC (G.depth, count_new_page_tables);
+ }
+
void
ggc_pch_read (f, addr)
FILE *f;
*************** ggc_pch_read (f, addr)
*** 1785,1793 ****
struct ggc_pch_ondisk d;
unsigned i;
char *offs = addr;
!
! /* We've just read in a PCH file. So, every object that used to be allocated
! is now free. */
clear_marks ();
#ifdef GGC_POISON
poison_pages ();
--- 2151,2163 ----
struct ggc_pch_ondisk d;
unsigned i;
char *offs = addr;
! unsigned long count_old_page_tables;
! unsigned long count_new_page_tables;
!
! count_old_page_tables = GGC_VARRAY_ACTIVE_SIZE (G.by_depth);
!
! /* We've just read in a PCH file. So, every object that used to be
! allocated is now free. */
clear_marks ();
#ifdef GGC_POISON
poison_pages ();
*************** ggc_pch_read (f, addr)
*** 1818,1827 ****
size_t bytes;
size_t num_objs;
size_t j;
!
if (d.totals[i] == 0)
continue;
!
bytes = ROUND_UP (d.totals[i] * OBJECT_SIZE (i), G.pagesize);
num_objs = bytes / OBJECT_SIZE (i);
entry = xcalloc (1, (sizeof (struct page_entry)
--- 2188,2197 ----
size_t bytes;
size_t num_objs;
size_t j;
!
if (d.totals[i] == 0)
continue;
!
bytes = ROUND_UP (d.totals[i] * OBJECT_SIZE (i), G.pagesize);
num_objs = bytes / OBJECT_SIZE (i);
entry = xcalloc (1, (sizeof (struct page_entry)
*************** ggc_pch_read (f, addr)
*** 1852,1858 ****
--- 2222,2243 ----
else
G.pages[i] = entry;
G.page_tails[i] = entry;
+
+ /* We start off by just adding all the new information to the
+ end of the varrays, later, we will move the new information
+ to the front of the varrays, as the PCH page tables are at
+ context 0. */
+ GGC_VARRAY_PUSH_GENERIC_PTRNGC (G.by_depth, entry);
+ GGC_VARRAY_PUSH_GENERIC_PTRNGC (G.save_in_use, 0);
}
+
+ /* Now, we update the various data structures that speed page table
+ handling. */
+
+ count_new_page_tables = GGC_VARRAY_ACTIVE_SIZE (G.by_depth)
+ - count_old_page_tables;
+
+ move_ptes_to_front (count_old_page_tables, count_new_page_tables);
/* Update the statistics. */
G.allocated = G.allocated_last_gc = offs - (char *)addr;