This is the mail archive of the
gcc-patches@gcc.gnu.org
mailing list for the GCC project.
Re: [PATCH][LTO] Fix PR41858
On Thu, 29 Oct 2009, Richard Guenther wrote:
>
> PR41858 shows that we are keeping too many files open at a time
> (all object files fed to lto1). This causes us to either fail or
> crash. The following patch avoids this by simply open/close the
> files whenever we read or mmap from them. The overhead of this
> should be way less than mmap overhead.
>
> I refrained from adding a 4000 file testcase ;)
>
> Bootstrapped and tested on x86_64-unknown-linux-gnu, ok for trunk?
This is a slightly revised patch which Ian approved on IRC. It
adds a single-entry file-descriptor cache and a FIXME notice.
Bootstrapped and tested on x86_64-unknown-linux-gnu, applied to trunk.
Richard.
2009-10-30 Richard Guenther <rguenther@suse.de>
PR lto/41858
* lto-streamer.h (struct lto_file_decl_data): Remove fd member.
lto/
* lto.c (lto_file_read): Do not set file_data->fd.
(lto_read_section_data): Use a single-entry file-descriptor cache.
Do not check the result of xmalloc.
(free_section_data): Do not use file_data->fd.
(lto_read_all_file_options): Likewise.
Index: gcc/lto/lto.c
===================================================================
*** gcc/lto/lto.c.orig 2009-10-30 15:21:07.000000000 +0100
--- gcc/lto/lto.c 2009-10-30 15:44:13.000000000 +0100
*************** lto_file_read (lto_file *file, FILE *res
*** 334,340 ****
file_data = XCNEW (struct lto_file_decl_data);
file_data->file_name = file->filename;
- file_data->fd = -1;
file_data->section_hash_table = lto_elf_build_section_table (file);
file_data->renaming_hash_table = lto_create_renaming_table ();
--- 334,339 ----
*************** lto_read_section_data (struct lto_file_d
*** 363,379 ****
intptr_t offset, size_t len)
{
char *result;
#if LTO_MMAP_IO
intptr_t computed_len;
intptr_t computed_offset;
intptr_t diff;
#endif
! if (file_data->fd == -1)
! file_data->fd = open (file_data->file_name, O_RDONLY);
!
! if (file_data->fd == -1)
! return NULL;
#if LTO_MMAP_IO
if (!page_mask)
--- 362,394 ----
intptr_t offset, size_t len)
{
char *result;
+ static int fd = -1;
+ static char *fd_name;
#if LTO_MMAP_IO
intptr_t computed_len;
intptr_t computed_offset;
intptr_t diff;
#endif
! /* Keep a single-entry file-descriptor cache. The last file we
! touched will get closed at exit.
! ??? Eventually we want to add a more sophisticated larger cache
! or rather fix function body streaming to not stream them in
! practically random order. */
! if (fd != -1
! && strcmp (fd_name, file_data->file_name) != 0)
! {
! free (fd_name);
! close (fd);
! fd = -1;
! }
! if (fd == -1)
! {
! fd_name = xstrdup (file_data->file_name);
! fd = open (file_data->file_name, O_RDONLY);
! if (fd == -1)
! return NULL;
! }
#if LTO_MMAP_IO
if (!page_mask)
*************** lto_read_section_data (struct lto_file_d
*** 387,412 ****
computed_len = len + diff;
result = (char *) mmap (NULL, computed_len, PROT_READ, MAP_PRIVATE,
! file_data->fd, computed_offset);
if (result == MAP_FAILED)
! {
! close (file_data->fd);
! return NULL;
! }
return result + diff;
#else
result = (char *) xmalloc (len);
! if (result == NULL)
! {
! close (file_data->fd);
! return NULL;
! }
! if (lseek (file_data->fd, offset, SEEK_SET) != offset
! || read (file_data->fd, result, len) != (ssize_t) len)
{
free (result);
- close (file_data->fd);
return NULL;
}
--- 402,418 ----
computed_len = len + diff;
result = (char *) mmap (NULL, computed_len, PROT_READ, MAP_PRIVATE,
! fd, computed_offset);
if (result == MAP_FAILED)
! return NULL;
return result + diff;
#else
result = (char *) xmalloc (len);
! if (lseek (fd, offset, SEEK_SET) != offset
! || read (fd, result, len) != (ssize_t) len)
{
free (result);
return NULL;
}
*************** get_section_data (struct lto_file_decl_d
*** 449,455 ****
starts at OFFSET and has LEN bytes. */
static void
! free_section_data (struct lto_file_decl_data *file_data,
enum lto_section_type section_type ATTRIBUTE_UNUSED,
const char *name ATTRIBUTE_UNUSED,
const char *offset, size_t len ATTRIBUTE_UNUSED)
--- 455,461 ----
starts at OFFSET and has LEN bytes. */
static void
! free_section_data (struct lto_file_decl_data *file_data ATTRIBUTE_UNUSED,
enum lto_section_type section_type ATTRIBUTE_UNUSED,
const char *name ATTRIBUTE_UNUSED,
const char *offset, size_t len ATTRIBUTE_UNUSED)
*************** free_section_data (struct lto_file_decl_
*** 460,468 ****
intptr_t diff;
#endif
- if (file_data->fd == -1)
- return;
-
#if LTO_MMAP_IO
computed_offset = ((intptr_t) offset) & page_mask;
diff = (intptr_t) offset - computed_offset;
--- 466,471 ----
*************** lto_read_all_file_options (void)
*** 1712,1726 ****
file_data = XCNEW (struct lto_file_decl_data);
file_data->file_name = file->filename;
- file_data->fd = -1;
file_data->section_hash_table = lto_elf_build_section_table (file);
lto_read_file_options (file_data);
lto_elf_file_close (file);
htab_delete (file_data->section_hash_table);
- if (file_data->fd != -1)
- close (file_data->fd);
free (file_data);
}
--- 1715,1726 ----
Index: gcc/lto-streamer.h
===================================================================
*** gcc/lto-streamer.h.orig 2009-10-24 00:54:59.000000000 +0200
--- gcc/lto-streamer.h 2009-10-30 15:21:20.000000000 +0100
*************** struct lto_file_decl_data
*** 555,564 ****
/* Nonzero if this file should be recompiled with LTRANS. */
unsigned needs_ltrans_p : 1;
- /* If the file is open, this is the fd of the mapped section. This
- is -1 if the file has not yet been opened. */
- int fd;
-
/* Hash table maps lto-related section names to location in file. */
htab_t section_hash_table;
--- 555,560 ----