* ggc-page.c (struct page_entry): Add index_by_depth field. Remove save_in_use_p field. (struct globals): Add by_depth, depth, and save_in_use fields. (INITIAL_PTE_COUNT): Add. (save_in_use_p_i): Add. (save_in_use_p): Add. (adjust_depth): Add. (move_ptes_to_front): Add. (free_page): Add support for and use faster data structures. (ggc_alloc): Likewise. (init_ggc): Likewise. (ggc_recalculate_in_use_p): Likewise. (ggc_pop_context): Likewise. (clear_marks): Likewise. (ggc_pch_read): Likewise. (ggc_varray_type): Add. (ggc_varray_head_tag): Add. (ggc_varray_data_enum): Add. (GGC_VARRAY_DATA_C): Add. (GGC_VARRAY_DATA_GENERICNGC): Add. (GGC_VARRAY_DATA_UNGC): Add. (GGC_NUM_VARRAY_DATA): Add. (union ggc_varray_data_tag): Add. (ggc_varray_data): Add. (ggc_varray_init): Add. (GGC_VARRAY_GENERIC_PTRNGC_INIT): Add. (GGC_VARRAY_UINTNGC_INIT): Add. (GGC_VARRAY_GENERIC_PTRNGC): Add. (GGC_VARRAY_UINTNGC): Add. (GGC_VARRAY_PUSH_GENERIC_PTRNGC): Add. (GGC_VARRAY_PUSH_UINTNGC): Add. (GGC_VARRAY_TOP_GENERIC_PTRNGC): Add. (GGC_VARRAY_TOP_UINTNGC): Add. (GGC_VARRAY_FREE): Add. (GGC_VARRAY_GROW): Add. (GGC_VARRAY_PUSH): Add. (GGC_VARRAY_POP): Add. (GGC_VARRAY_TOP): Add. (GGC_VARRAY_HDR_SIZE): Add. (ggc_varray_init): Add. (ggc_varray_grow): Add. (ggc_varray_check_failed): Add. (GGC_VARRAY_ACTIVE_SIZE): Add. * varray.c (element_size): Remove. (uses_ggc): Remove. (element): Add. (varray_init): Use new interface. (varray_grow): Use new interface. (varray_clear): Use new interface. Index: varray.c =================================================================== RCS file: /cvs/gcc/gcc/gcc/varray.c,v retrieving revision 1.16 diff -p -r1.16 varray.c *** varray.c 16 Dec 2002 18:20:02 -0000 1.16 --- varray.c 6 Feb 2003 01:54:07 -0000 *************** *** 29,62 **** #define VARRAY_HDR_SIZE (sizeof (struct varray_head_tag) - sizeof (varray_data)) ! static const size_t element_size[NUM_VARRAY_DATA] = { ! sizeof (char), ! sizeof (unsigned char), ! sizeof (short), ! sizeof (unsigned short), ! sizeof (int), ! sizeof (unsigned int), ! sizeof (long), ! sizeof (unsigned long), ! sizeof (HOST_WIDE_INT), ! sizeof (unsigned HOST_WIDE_INT), ! sizeof (PTR), ! sizeof (char *), ! sizeof (struct rtx_def *), ! sizeof (struct rtvec_def *), ! sizeof (union tree_node *), ! sizeof (struct bitmap_head_def *), ! sizeof (struct reg_info_def *), ! sizeof (struct const_equiv_data), ! sizeof (struct basic_block_def *), ! sizeof (struct elt_list *) ! }; ! static const int uses_ggc[NUM_VARRAY_DATA] = { ! 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* unsigned HOST_WIDE_INT */ ! 1, /* PTR */ ! 1, 1, 1, 1, 1, /* bitmap_head_def */ ! 0, 0, 0, 1 }; /* Allocate a virtual array with NUM_ELEMENT elements, each of which is --- 29,61 ---- #define VARRAY_HDR_SIZE (sizeof (struct varray_head_tag) - sizeof (varray_data)) ! /* Do not add any more non-GC items here. Please either remove or GC those items that ! are not GCed. */ ! static const struct { ! size_t size; ! int uses_ggc; ! } element[NUM_VARRAY_DATA] = { ! { sizeof (char), 1 }, ! { sizeof (unsigned char), 1 }, ! { sizeof (short), 1 }, ! { sizeof (unsigned short), 1 }, ! { sizeof (int), 1 }, ! { sizeof (unsigned int), 1 }, ! { sizeof (long), 1 }, ! { sizeof (unsigned long), 1 }, ! { sizeof (HOST_WIDE_INT), 1 }, ! { sizeof (unsigned HOST_WIDE_INT), 1 }, ! { sizeof (PTR), 1 }, ! { sizeof (char *), 1 }, ! { sizeof (struct rtx_def *), 1 }, ! { sizeof (struct rtvec_def *), 1 }, ! { sizeof (union tree_node *), 1 }, ! { sizeof (struct bitmap_head_def *), 1 }, ! { sizeof (struct reg_info_def *), 0 }, ! { sizeof (struct const_equiv_data), 0 }, ! { sizeof (struct basic_block_def *), 0 }, ! { sizeof (struct elt_list *), 1 }, }; /* Allocate a virtual array with NUM_ELEMENT elements, each of which is *************** varray_init (num_elements, element_kind, *** 67,75 **** enum varray_data_enum element_kind; const char *name; { ! size_t data_size = num_elements * element_size[element_kind]; varray_type ptr; ! if (uses_ggc [element_kind]) ptr = (varray_type) ggc_alloc_cleared (VARRAY_HDR_SIZE + data_size); else ptr = (varray_type) xcalloc (VARRAY_HDR_SIZE + data_size, 1); --- 66,74 ---- enum varray_data_enum element_kind; const char *name; { ! size_t data_size = num_elements * element[element_kind].size; varray_type ptr; ! if (element[element_kind].uses_ggc) ptr = (varray_type) ggc_alloc_cleared (VARRAY_HDR_SIZE + data_size); else ptr = (varray_type) xcalloc (VARRAY_HDR_SIZE + data_size, 1); *************** varray_grow (va, n) *** 92,102 **** if (n != old_elements) { ! size_t elem_size = element_size[va->type]; size_t old_data_size = old_elements * elem_size; size_t data_size = n * elem_size; ! if (uses_ggc[va->type]) va = (varray_type) ggc_realloc (va, VARRAY_HDR_SIZE + data_size); else va = (varray_type) xrealloc ((char *) va, VARRAY_HDR_SIZE + data_size); --- 91,101 ---- if (n != old_elements) { ! size_t elem_size = element[va->type].size; size_t old_data_size = old_elements * elem_size; size_t data_size = n * elem_size; ! if (element[va->type].uses_ggc) va = (varray_type) ggc_realloc (va, VARRAY_HDR_SIZE + data_size); else va = (varray_type) xrealloc ((char *) va, VARRAY_HDR_SIZE + data_size); *************** void *** 113,119 **** varray_clear (va) varray_type va; { ! size_t data_size = element_size[va->type] * va->num_elements; memset (va->data.c, 0, data_size); va->elements_used = 0; --- 112,118 ---- varray_clear (va) varray_type va; { ! size_t data_size = element[va->type].size * va->num_elements; memset (va->data.c, 0, data_size); va->elements_used = 0; Index: varray.h =================================================================== RCS file: /cvs/gcc/gcc/gcc/varray.h,v retrieving revision 1.29 diff -p -r1.29 varray.h *** varray.h 16 Dec 2002 18:20:02 -0000 1.29 --- varray.h 6 Feb 2003 01:54:07 -0000 *************** struct const_equiv_data GTY(()) { *** 57,63 **** }; /* Enum indicating what the varray contains. ! If this is changed, `element_size' in varray.c needs to be updated. */ enum varray_data_enum { VARRAY_DATA_C, --- 57,63 ---- }; /* Enum indicating what the varray contains. ! If this is changed, `element' in varray.c needs to be updated. */ enum varray_data_enum { VARRAY_DATA_C, Index: ggc-page.c =================================================================== RCS file: /cvs/gcc/gcc/gcc/ggc-page.c,v retrieving revision 1.61 diff -p -r1.61 ggc-page.c *** ggc-page.c 30 Jan 2003 18:14:06 -0000 1.61 --- ggc-page.c 6 Feb 2003 01:54:09 -0000 *************** Software Foundation, 59 Temple Place - S *** 26,32 **** #include "rtl.h" #include "tm_p.h" #include "toplev.h" - #include "varray.h" #include "flags.h" #include "ggc.h" #include "timevar.h" --- 26,31 ---- *************** typedef struct page_entry *** 253,261 **** struct page_group *group; #endif ! /* Saved in-use bit vector for pages that aren't in the topmost ! context during collection. */ ! unsigned long *save_in_use_p; /* Context depth of this page. */ unsigned short context_depth; --- 252,260 ---- struct page_group *group; #endif ! /* This is the index in the by_depth varray where this page table ! can be found. */ ! unsigned long index_by_depth; /* Context depth of this page. */ unsigned short context_depth; *************** typedef struct page_table_chain *** 314,319 **** --- 313,320 ---- #endif + typedef struct ggc_varray_head_tag *ggc_varray_type; + /* The rest of the global variables. */ static struct globals { *************** static struct globals *** 367,372 **** --- 368,391 ---- /* The file descriptor for debugging output. */ FILE *debug_file; + + /* Each element of this array is a page_entry, all page_entries can + be found in here by increasing depth. index_by_depth in the + page_entry is the index into this data structure where that + page_entry can be found. This is used to speed up finding all + page_entries at a particular depth. */ + ggc_varray_type by_depth; + + /* Each element is an index in by_depth where the given depth + starts. This structure is indexed by that given depth we are + interested in. */ + ggc_varray_type depth; + + /* Each element is a pointer to the saved in_use_p bits, if any, + zero otherwise. We allocate them all together, to enable a + better runtime data access pattern. */ + ggc_varray_type save_in_use; + } G; /* The size in bytes required to maintain a bitmap for the objects *************** static struct globals *** 379,384 **** --- 398,406 ---- free list. This cannot be larger than HOST_BITS_PER_INT for the in_use bitmask for page_group. */ #define GGC_QUIRE_SIZE 16 + + /* Initial guess as to how many page table entries we might need. */ + #define INITIAL_PTE_COUNT 128 static int ggc_allocated_p PARAMS ((const void *)); static page_entry *lookup_page_table_entry PARAMS ((const void *)); *************** static void clear_marks PARAMS ((void)); *** 398,403 **** --- 420,427 ---- static void sweep_pages PARAMS ((void)); static void ggc_recalculate_in_use_p PARAMS ((page_entry *)); static void compute_inverse PARAMS ((unsigned)); + static inline void adjust_depth PARAMS ((void)); + static void move_ptes_to_front PARAMS ((int, int)); #ifdef ENABLE_GC_CHECKING static void poison_pages PARAMS ((void)); *************** static void poison_pages PARAMS ((void)) *** 405,410 **** --- 429,597 ---- void debug_print_page_list PARAMS ((int)); + /* Internal varray management code. This is separated from varrays to + enable the removal of non-GCed objects from varray.[ch] */ + + enum ggc_varray_data_enum { + GGC_VARRAY_DATA_C, + GGC_VARRAY_DATA_GENERICNGC, + GGC_VARRAY_DATA_UNGC, + GGC_NUM_VARRAY_DATA + }; + + /* Union of various array types that are used. */ + typedef union ggc_varray_data_tag { + char c[1]; + PTR genericngc[1]; + unsigned int ungc[1]; + } ggc_varray_data; + + /* Virtual array of pointers header. */ + struct ggc_varray_head_tag { + size_t num_elements; /* Maximum element number allocated. */ + size_t elements_used; /* The number of elements used, if + using GGC_VARRAY_PUSH/GGC_VARRAY_POP. */ + enum ggc_varray_data_enum type; /* The kind of elements in the varray. */ + const char *name; /* name of the varray for reporting errors */ + ggc_varray_data data; /* The data elements follow, must be last. */ + }; + + /* Allocate a virtual array with NUM elements, each of which is SIZE bytes + long, named NAME. Array elements are zeroed. */ + static ggc_varray_type ggc_varray_init PARAMS ((size_t, enum ggc_varray_data_enum, + const char *)); + + #define GGC_VARRAY_GENERIC_PTRNGC_INIT(va, num, name) \ + va = ggc_varray_init (num, GGC_VARRAY_DATA_GENERICNGC, name) + + #define GGC_VARRAY_UINTNGC_INIT(va, num, name) \ + va = ggc_varray_init (num, GGC_VARRAY_DATA_UNGC, name) + + #define GGC_VARRAY_GENERIC_PTRNGC(VA,N) GGC_VARRAY_CHECK (VA, N, genericngc) + #define GGC_VARRAY_UINTNGC(VA, N) GGC_VARRAY_CHECK (VA, N, ungc) + + /* Free up memory allocated by the virtual array, but do not free any of the + elements involved. */ + #define GGC_VARRAY_FREE(vp) \ + do { if (vp) { free (vp); vp = (ggc_varray_type) 0; } } while (0) + #define GGC_VARRAY_GROW(VA, N) ((VA) = ggc_varray_grow (VA, N)) + + /* Push X onto VA. T is the name of the field in varray_data + corresponding to the type of X. */ + #define GGC_VARRAY_PUSH(VA, T, X) \ + do \ + { \ + if ((VA)->elements_used >= (VA)->num_elements) \ + GGC_VARRAY_GROW ((VA), 2 * (VA)->num_elements); \ + (VA)->data.T[(VA)->elements_used++] = (X); \ + } \ + while (0) + + #define GGC_VARRAY_PUSH_GENERIC_PTRNGC(VA, X) GGC_VARRAY_PUSH (VA, genericngc, X) + #define GGC_VARRAY_PUSH_UINTNGC(VA, X) GGC_VARRAY_PUSH (VA, ungc, X) + + /* Pop the top element of VA. */ + #define GGC_VARRAY_POP(VA) \ + ((VA)->elements_used--) + + /* Return the top element of VA. */ + #define GGC_VARRAY_TOP(VA, T) \ + ((VA)->data.T[(VA)->elements_used - 1]) + #define GGC_VARRAY_TOP_GENERIC_PTRNGC(VA) GGC_VARRAY_TOP (VA, genericngc) + #define GGC_VARRAY_TOP_UINTNGC(VA) GGC_VARRAY_TOP (VA, ungc) + + #define GGC_VARRAY_HDR_SIZE (sizeof (struct ggc_varray_head_tag) - sizeof (ggc_varray_data)) + + static const size_t element[GGC_NUM_VARRAY_DATA] = { + sizeof (char), + sizeof (PTR), + sizeof (unsigned int) + }; + + static inline ggc_varray_type ggc_varray_grow PARAMS ((ggc_varray_type, size_t)); + + /* Allocate a virtual array with NUM_ELEMENT elements, each of which is + ELEMENT_SIZE bytes long, named NAME. Array elements are zeroed. */ + static ggc_varray_type + ggc_varray_init (num_elements, element_kind, name) + size_t num_elements; + enum ggc_varray_data_enum element_kind; + const char *name; + { + size_t data_size = num_elements * element[element_kind]; + ggc_varray_type ptr; + ptr = (ggc_varray_type) xcalloc (GGC_VARRAY_HDR_SIZE + data_size, 1); + + ptr->num_elements = num_elements; + ptr->elements_used = 0; + ptr->type = element_kind; + ptr->name = name; + return ptr; + } + + /* Grow/shrink the virtual array VA to N elements. Zero any new elements + allocated. */ + static inline ggc_varray_type + ggc_varray_grow (va, n) + ggc_varray_type va; + size_t n; + { + size_t old_elements = va->num_elements; + + if (n != old_elements) + { + size_t elem_size = element[va->type]; + size_t old_data_size = old_elements * elem_size; + size_t data_size = n * elem_size; + + va = (ggc_varray_type) xrealloc ((char *) va, GGC_VARRAY_HDR_SIZE + data_size); + va->num_elements = n; + if (n > old_elements) + memset (&va->data.c[old_data_size], 0, data_size - old_data_size); + } + + return va; + } + + /* Check the bounds of a varray access. */ + + #if defined ENABLE_CHECKING && (GCC_VERSION >= 2007) + extern void ggc_varray_check_failed PARAMS ((ggc_varray_type, size_t, + const char *, int, + const char *)) ATTRIBUTE_NORETURN; + #define GGC_VARRAY_CHECK(VA, N, T) __extension__ \ + (*({ ggc_varray_type const _va = (VA); \ + const size_t _n = (N); \ + if (_n >= _va->num_elements) \ + ggc_varray_check_failed (_va, _n, __FILE__, __LINE__, __FUNCTION__); \ + &_va->data.T[_n]; })) + + extern void error PARAMS ((const char *, ...)) ATTRIBUTE_PRINTF_1; + + void + ggc_varray_check_failed (va, n, file, line, function) + ggc_varray_type va; + size_t n; + const char *file; + int line; + const char *function; + { + internal_error ("virtual array %s[%lu]: element %lu out of bounds in %s, at %s:%d", + va->name, (unsigned long) va->num_elements, (unsigned long) n, + function, trim_filename (file), line); + } + #else + #define GGC_VARRAY_CHECK(VA, N, T) ((VA)->data.T[N]) + #endif + + #define GGC_VARRAY_ACTIVE_SIZE(VA) ((VA)->elements_used) + + + #define save_in_use_p_i(__i) \ + (*(unsigned long **)&GGC_VARRAY_GENERIC_PTRNGC (G.save_in_use, __i)) + #define save_in_use_p(__p) \ + (save_in_use_p_i (__p->index_by_depth)) + /* Returns nonzero if P was allocated in GC'able memory. */ static inline int *************** alloc_page (order) *** 772,777 **** --- 959,983 ---- return entry; } + /* Adjust the size of G.depth so that no index greater than the one + used by the top of the G.by_depth varray is used. */ + + static inline void + adjust_depth () + { + page_entry *top; + + if (GGC_VARRAY_ACTIVE_SIZE (G.by_depth)) + { + top = GGC_VARRAY_TOP_GENERIC_PTRNGC (G.by_depth); + /* Peel back indicies in depth that index into by_depth, so that + as new elements are added to by_depth, we note the indicies + of those elements, if they are for new context depths. */ + while (GGC_VARRAY_ACTIVE_SIZE (G.depth) > (size_t)top->context_depth+1) + GGC_VARRAY_POP (G.depth); + } + } + /* For a page that is no longer needed, put it on the free page list. */ static inline void *************** free_page (entry) *** 793,798 **** --- 999,1030 ---- clear_page_group_in_use (entry->group, entry->page); #endif + if (GGC_VARRAY_ACTIVE_SIZE (G.by_depth) > 1) + { + page_entry *top = GGC_VARRAY_TOP_GENERIC_PTRNGC (G.by_depth); + + /* If they are at the same depth, put top element into freed + slot. */ + if (entry->context_depth == top->context_depth) + { + int i = entry->index_by_depth; + GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, i) = top; + GGC_VARRAY_GENERIC_PTRNGC (G.save_in_use, i) + = GGC_VARRAY_TOP_GENERIC_PTRNGC (G.save_in_use); + top->index_by_depth = i; + } + else + { + /* We cannot free a page from a context deeper than the + current one. */ + abort (); + } + } + GGC_VARRAY_POP (G.by_depth); + GGC_VARRAY_POP (G.save_in_use); + + adjust_depth (); + entry->next = G.free_pages; G.free_pages = entry; } *************** ggc_alloc (size) *** 916,921 **** --- 1148,1162 ---- struct page_entry *new_entry; new_entry = alloc_page (order); + new_entry->index_by_depth = GGC_VARRAY_ACTIVE_SIZE (G.by_depth); + GGC_VARRAY_PUSH_GENERIC_PTRNGC (G.by_depth, new_entry); + GGC_VARRAY_PUSH_GENERIC_PTRNGC (G.save_in_use, 0); + + /* We can skip context depths, if we do, make sure we go all the + way to the new depth. */ + while (new_entry->context_depth >= GGC_VARRAY_ACTIVE_SIZE (G.depth)) + GGC_VARRAY_PUSH_UINTNGC (G.depth, GGC_VARRAY_ACTIVE_SIZE (G.by_depth)-1); + /* If this is the only entry, it's also the tail. */ if (entry == NULL) G.page_tails[order] = new_entry; *************** init_ggc () *** 1218,1223 **** --- 1459,1468 ---- for (i = OBJECT_SIZE (order); size_lookup [i] == o; --i) size_lookup[i] = order; } + + GGC_VARRAY_GENERIC_PTRNGC_INIT (G.by_depth, INITIAL_PTE_COUNT, "by_depth"); + GGC_VARRAY_GENERIC_PTRNGC_INIT (G.save_in_use, INITIAL_PTE_COUNT, "save_in_use"); + GGC_VARRAY_UINTNGC_INIT (G.depth, 10, "depth"); } /* Increment the `GC context'. Objects allocated in an outer context *************** ggc_recalculate_in_use_p (p) *** 1260,1266 **** /* Something is in use if it is marked, or if it was in use in a context further down the context stack. */ ! p->in_use_p[i] |= p->save_in_use_p[i]; /* Decrement the free object count for every object allocated. */ for (j = p->in_use_p[i]; j; j >>= 1) --- 1505,1511 ---- /* Something is in use if it is marked, or if it was in use in a context further down the context stack. */ ! p->in_use_p[i] |= save_in_use_p (p)[i]; /* Decrement the free object count for every object allocated. */ for (j = p->in_use_p[i]; j; j >>= 1) *************** void *** 1278,1284 **** ggc_pop_context () { unsigned long omask; ! unsigned order, depth; depth = --G.context_depth; omask = (unsigned long)1 << (depth + 1); --- 1523,1532 ---- ggc_pop_context () { unsigned long omask; ! unsigned depth, i, e; ! #ifdef ENABLE_CHECKING ! unsigned order; ! #endif depth = --G.context_depth; omask = (unsigned long)1 << (depth + 1); *************** ggc_pop_context () *** 1290,1298 **** G.context_depth_allocations &= omask - 1; G.context_depth_collections &= omask - 1; ! /* Any remaining pages in the popped context are lowered to the new ! current context; i.e. objects allocated in the popped context and ! left over are imported into the previous context. */ for (order = 2; order < NUM_ORDERS; order++) { page_entry *p; --- 1538,1603 ---- G.context_depth_allocations &= omask - 1; G.context_depth_collections &= omask - 1; ! /* The G.depth array is shortend so that the last index is the ! context_depth of the top element of by_depth. */ ! if (depth+1 < GGC_VARRAY_ACTIVE_SIZE (G.depth)) ! e = GGC_VARRAY_UINTNGC (G.depth, depth+1); ! else ! e = GGC_VARRAY_ACTIVE_SIZE (G.by_depth); ! ! /* We might not have any PTEs of depth depth. */ ! if (depth < GGC_VARRAY_ACTIVE_SIZE (G.depth)) ! { ! ! /* First we go through all the pages at depth depth to ! recalculate the in use bits. */ ! for (i = GGC_VARRAY_UINTNGC (G.depth, depth); i < e; ++i) ! { ! page_entry *p; ! ! #ifdef ENABLE_CHECKING ! p = GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, i); ! ! /* Check that all of the pages really are at the depth that ! we expect. */ ! if (p->context_depth != depth) ! abort (); ! if (p->index_by_depth != i) ! abort (); ! #endif ! ! __builtin_prefetch (&save_in_use_p_i (i+8)); ! __builtin_prefetch (&save_in_use_p_i (i+16)); ! if (save_in_use_p_i (i)) ! { ! p = GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, i); ! ggc_recalculate_in_use_p (p); ! free (save_in_use_p_i (i)); ! save_in_use_p_i (i) = 0; ! } ! } ! } ! ! /* Then, we reset all page_entries with a depth greater than depth ! to be at depth. */ ! for (i = e; i < GGC_VARRAY_ACTIVE_SIZE (G.by_depth); ++i) ! { ! page_entry *p = GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, i); ! ! /* Check that all of the pages really are at the depth we ! expect. */ ! #ifdef ENABLE_CHECKING ! if (p->context_depth <= depth) ! abort (); ! if (p->index_by_depth != i) ! abort (); ! #endif ! p->context_depth = depth; ! } ! ! adjust_depth (); ! ! #ifdef ENABLE_CHECKING for (order = 2; order < NUM_ORDERS; order++) { page_entry *p; *************** ggc_pop_context () *** 1300,1317 **** for (p = G.pages[order]; p != NULL; p = p->next) { if (p->context_depth > depth) ! p->context_depth = depth; ! ! /* If this page is now in the topmost context, and we'd ! saved its allocation state, restore it. */ ! else if (p->context_depth == depth && p->save_in_use_p) ! { ! ggc_recalculate_in_use_p (p); ! free (p->save_in_use_p); ! p->save_in_use_p = 0; ! } } } } /* Unmark all objects. */ --- 1605,1616 ---- for (p = G.pages[order]; p != NULL; p = p->next) { if (p->context_depth > depth) ! abort (); ! else if (p->context_depth == depth && save_in_use_p (p)) ! abort (); } } + #endif } /* Unmark all objects. */ *************** clear_marks () *** 1341,1349 **** marks. So, back them up first. */ if (p->context_depth < G.context_depth) { ! if (! p->save_in_use_p) ! p->save_in_use_p = xmalloc (bitmap_size); ! memcpy (p->save_in_use_p, p->in_use_p, bitmap_size); } /* Reset reset the number of free objects and clear the --- 1640,1648 ---- marks. So, back them up first. */ if (p->context_depth < G.context_depth) { ! if (! save_in_use_p (p)) ! save_in_use_p (p) = xmalloc (bitmap_size); ! memcpy (save_in_use_p (p), p->in_use_p, bitmap_size); } /* Reset reset the number of free objects and clear the *************** ggc_pch_finish (d, f) *** 1777,1782 **** --- 2076,2148 ---- free (d); } + /* Move the PCH PTE entries just added to the end of by_depth, to the + front. */ + + static void + move_ptes_to_front (count_old_page_tables, count_new_page_tables) + int count_old_page_tables; + int count_new_page_tables; + { + unsigned i; + + /* First, we swap the new entries to the front of the varrays. */ + + ggc_varray_type new_by_depth; + ggc_varray_type new_save_in_use; + int c; + + c = count_old_page_tables + count_new_page_tables; + /* Allow for at least 25% growth without resizing. */ + c = c + c/4; + c = MAX (c, INITIAL_PTE_COUNT); + + GGC_VARRAY_GENERIC_PTRNGC_INIT (new_by_depth, + c, + "by_depth"); + GGC_VARRAY_GENERIC_PTRNGC_INIT (new_save_in_use, + c, + "save_in_use"); + GGC_VARRAY_ACTIVE_SIZE (new_by_depth) = count_old_page_tables + + count_new_page_tables; + GGC_VARRAY_ACTIVE_SIZE (new_save_in_use) = count_old_page_tables + + count_new_page_tables; + + memcpy (&GGC_VARRAY_GENERIC_PTRNGC (new_by_depth, 0), + &GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, count_old_page_tables), + count_new_page_tables * sizeof (void *)); + memcpy (&GGC_VARRAY_GENERIC_PTRNGC (new_by_depth, count_new_page_tables), + &GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, 0), + count_old_page_tables * sizeof (void *)); + memcpy (&GGC_VARRAY_GENERIC_PTRNGC (new_save_in_use, 0), + &GGC_VARRAY_GENERIC_PTRNGC (G.save_in_use, count_old_page_tables), + count_new_page_tables * sizeof (void *)); + memcpy (&GGC_VARRAY_GENERIC_PTRNGC (new_save_in_use, count_new_page_tables), + &GGC_VARRAY_GENERIC_PTRNGC (G.save_in_use, 0), + count_old_page_tables * sizeof (void *)); + + GGC_VARRAY_FREE (G.by_depth); + GGC_VARRAY_FREE (G.save_in_use); + + G.by_depth = new_by_depth; + G.save_in_use = new_save_in_use; + + /* Now update all the index_by_depth fields. */ + for (i = GGC_VARRAY_ACTIVE_SIZE (G.by_depth); i > 0; --i) + { + page_entry *p = GGC_VARRAY_GENERIC_PTRNGC (G.by_depth, i-1); + p->index_by_depth = i-1; + } + + /* And last, we update the depth pointers in G.depth. The first + entry is already 0, and context 0 entries always start at index + 0, so there is nothing to update in the first slot. We need a + second slot, only if we have old ptes, and if we do, they start + at index count_new_page_tables. */ + if (count_old_page_tables) + GGC_VARRAY_PUSH_UINTNGC (G.depth, count_new_page_tables); + } + void ggc_pch_read (f, addr) FILE *f; *************** ggc_pch_read (f, addr) *** 1785,1793 **** struct ggc_pch_ondisk d; unsigned i; char *offs = addr; ! ! /* We've just read in a PCH file. So, every object that used to be allocated ! is now free. */ clear_marks (); #ifdef GGC_POISON poison_pages (); --- 2151,2163 ---- struct ggc_pch_ondisk d; unsigned i; char *offs = addr; ! unsigned long count_old_page_tables; ! unsigned long count_new_page_tables; ! ! count_old_page_tables = GGC_VARRAY_ACTIVE_SIZE (G.by_depth); ! ! /* We've just read in a PCH file. So, every object that used to be ! allocated is now free. */ clear_marks (); #ifdef GGC_POISON poison_pages (); *************** ggc_pch_read (f, addr) *** 1818,1827 **** size_t bytes; size_t num_objs; size_t j; ! if (d.totals[i] == 0) continue; ! bytes = ROUND_UP (d.totals[i] * OBJECT_SIZE (i), G.pagesize); num_objs = bytes / OBJECT_SIZE (i); entry = xcalloc (1, (sizeof (struct page_entry) --- 2188,2197 ---- size_t bytes; size_t num_objs; size_t j; ! if (d.totals[i] == 0) continue; ! bytes = ROUND_UP (d.totals[i] * OBJECT_SIZE (i), G.pagesize); num_objs = bytes / OBJECT_SIZE (i); entry = xcalloc (1, (sizeof (struct page_entry) *************** ggc_pch_read (f, addr) *** 1852,1858 **** --- 2222,2243 ---- else G.pages[i] = entry; G.page_tails[i] = entry; + + /* We start off by just adding all the new information to the + end of the varrays, later, we will move the new information + to the front of the varrays, as the PCH page tables are at + context 0. */ + GGC_VARRAY_PUSH_GENERIC_PTRNGC (G.by_depth, entry); + GGC_VARRAY_PUSH_GENERIC_PTRNGC (G.save_in_use, 0); } + + /* Now, we update the various data structures that speed page table + handling. */ + + count_new_page_tables = GGC_VARRAY_ACTIVE_SIZE (G.by_depth) + - count_old_page_tables; + + move_ptes_to_front (count_old_page_tables, count_new_page_tables); /* Update the statistics. */ G.allocated = G.allocated_last_gc = offs - (char *)addr;